Merge branch 'ToPython3' into dev

hitier
2 parents 8b29751b c269db3f
Showing 2 changed files with 66 additions and 56 deletions Show diff stats
web/run.py
web/view/home.html.jinja2
 # coding=utf-8
-import StringIO
+from io import StringIO
 import datetime
 import gzip
 import json
@@ -8,7 +8,7 @@ import logging
 import random
 import tarfile
 import time
-import urllib
+import urllib.request as urllib_request
 import requests
 import re  # regex
 import numpy
@@ -25,8 +25,13 @@ from dateutil import parser as dateparser
 from flask import Flask
 from flask import request
 from flask import url_for, send_from_directory, abort as abort_flask
-from jinja2 import Environment, FileSystemLoader, Markup
-from yaml import load as yaml_load
+from jinja2 import Environment, FileSystemLoader
+from jinja2.utils import markupsafe
+from yaml import load as yaml_load, dump
+try:
+    from yaml import CLoader as Loader
+except ImportError:
+    from yaml import Loader
 from netCDF4 import Dataset, date2num
@@ -48,7 +53,7 @@ with open(get_path(&#39;../VERSION&#39;), &#39;r&#39;) as version_file:
 # CONFIG
 with open(get_path('../config.yml'), 'r') as config_file:
-    config = yaml_load(config_file.read())
+    config = yaml_load(config_file.read(), Loader=Loader)
 FILE_DATE_FMT = "%Y-%m-%dT%H:%M:%S"
 MOMENT_DATE_FMT = "%Y-%m-%dT%H:%M:%SZ"
@@ -148,13 +153,11 @@ PARAMETERS = {
     },
 }
-
 # SETUP ENVIRONMENT ###########################################################
 environ['SPACEPY'] = CACHE_DIR
 environ['CDF_LIB'] = CDF_LIB
-
 # SETUP FLASK ENGINE ##########################################################
 app = Flask(__name__, root_path=THIS_DIRECTORY)
@@ -211,20 +214,20 @@ def markdown_filter(value, nl2br=False, p=True):
 _js_escapes = {
-        '\\': '\\u005C',
-        '\'': '\\u0027',
-        '"': '\\u0022',
-        '>': '\\u003E',
-        '<': '\\u003C',
-        '&': '\\u0026',
-        '=': '\\u003D',
-        '-': '\\u002D',
-        ';': '\\u003B',
-        u'\u2028': '\\u2028',
-        u'\u2029': '\\u2029'
+    '\\': '\\u005C',
+    '\'': '\\u0027',
+    '"': '\\u0022',
+    '>': '\\u003E',
+    '<': '\\u003C',
+    '&': '\\u0026',
+    '=': '\\u003D',
+    '-': '\\u002D',
+    ';': '\\u003B',
+    u'\u2028': '\\u2028',
+    u'\u2029': '\\u2029'
 }
 # Escape every ASCII character with a value less than 32.
-_js_escapes.update(('%c' % z, '\\u%04X' % z) for z in xrange(32))
+_js_escapes.update(('%c' % z, '\\u%04X' % z) for z in range(32))
 def escapejs_filter(value):
@@ -235,7 +238,7 @@ def escapejs_filter(value):
         else:
             escaped.append(letter)
-    return Markup("".join(escaped))
+    return markupsafe.Markup("".join(escaped))
 tpl_engine = Environment(loader=FileSystemLoader([get_path('view')]),
                          trim_blocks=True,
@@ -275,7 +278,7 @@ def render_view(view, context=None):
     """
     context = {} if context is None else context
     return tpl_engine.get_template(view).render(
-        dict(tpl_global_vars.items() + context.items())
+        dict(list(tpl_global_vars.items()) + list(context.items()))
     )
@@ -314,7 +317,7 @@ def round_time(dt=None, round_to=60):
         dt = datetime.datetime.now()
     seconds = (dt.replace(tzinfo=None) - dt.min).seconds
     rounding = (seconds + round_to / 2) // round_to * round_to
-    return dt + datetime.timedelta(0, rounding-seconds, -dt.microsecond)
+    return dt + datetime.timedelta(0, rounding - seconds, -dt.microsecond)
 def datetime_from_list(time_list):
@@ -459,10 +462,10 @@ ORDER BY time_min, granule_gid
     try:
         response = requests.post(api_url, {
             'REQUEST': 'doQuery',
-            'LANG':    'ADQL',
-            'QUERY':   query,
+            'LANG': 'ADQL',
+            'QUERY': query,
             'TIMEOUT': '30',
-            'FORMAT':  'VOTable/td'
+            'FORMAT': 'VOTable/td'
         })
         response_xml = response.text
@@ -516,7 +519,7 @@ def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at):
     remote_gzip_files = []
     while not success and retries < 3:
         try:
-            response = urllib.urlopen(url)
+            response = urllib_request.urlopen(url)
             remote_gzip_files = json.loads(response.read())
             if not remote_gzip_files:
                 raise Exception("Failed to fetch data at '%s'." % url)
@@ -529,7 +532,7 @@ def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at):
                 # raise Exception("API says it's out of time at '%s'." % url)
             success = True
         except Exception as e:
-            log.warn("Failed (%d/3) '%s' : %s" % (retries+1, url, e.message))
+            log.warning("Failed (%d/3) '%s' : %s" % (retries + 1, url, e.message))
             remote_gzip_files = []
             errors.append(e)
         finally:
@@ -537,9 +540,9 @@ def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at):
     if not remote_gzip_files:
         log.error("Failed to retrieve data from AMDA.")
         log.error("Failed to fetch gzip files list for %s at '%s' : %s" %
-                   (orbiter, url, errors))
+                  (orbiter, url, errors))
         abort(400, "Failed to fetch gzip files list for %s at '%s' : %s" %
-                   (orbiter, url, errors))
+              (orbiter, url, errors))
     else:
         remote_gzip_files = list(set(remote_gzip_files))
@@ -558,7 +561,7 @@ def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at):
         local_gzip_files.append(local_gzip_file)
         if not isfile(local_gzip_file):
             log.debug("Retrieving '%s'..." % local_gzip_file)
-            urllib.urlretrieve(remote_gzip_file, local_gzip_file)
+            urllib_request.urlretrieve(remote_gzip_file, local_gzip_file)
             log.debug("Retrieved '%s'." % local_gzip_file)
         else:
             log.debug("Found '%s' in the cache." % local_gzip_file)
@@ -708,13 +711,17 @@ def get_data_for_target(target_config, input_slug,
                       (target_config['name'], orbit_file))
             for ltime, datum_hee in zip(times, data_hee):
                 try:
+                    try:
+                        ltime = [str(i, 'UTF8') for i in ltime]
+                    except Exception as e:
+                        print(e)
                     dtime = datetime_from_list(ltime)
                 except Exception:
-                    log.error("Failed to parse time from %s." % ltime)
+                    log.error("Failed to parse time from get__data_for_target %s." % ltime)
                     raise
                 # Keep only what's in the interval
                 if s0 <= dtime <= s1:
-                    dkey = round_time(dtime, 60*60).strftime(precision)
+                    dkey = round_time(dtime, 60 * 60).strftime(precision)
                     orbit_data[dkey] = datum_hee
             cdf_handle.close()
@@ -730,8 +737,8 @@ def get_data_for_target(target_config, input_slug,
             nc_keys.update(model['parameters'])
         if len(model_files) == 0:
-            log.warn("No model data for '%s' '%s'."
-                     % (target_config['slug'], model['slug']))
+            log.warning("No model data for '%s' '%s'."
+                        % (target_config['slug'], model['slug']))
         for model_file in model_files:
             log.debug("%s: opening model NETCDF4 '%s'..." %
@@ -769,6 +776,10 @@ def get_data_for_target(target_config, input_slug,
                     in zip(times, data_v, data_b, data_t, data_n, data_p, data_a):
                 try:
+                    try:
+                        ltime = [str(i, 'UTF8') for i in ltime]
+                    except Exception as e:
+                        print(e)
                     dtime = datetime_from_list(ltime)
                 except Exception:
                     log.error("Failed to parse time from %s." % ltime)
@@ -777,7 +788,7 @@ def get_data_for_target(target_config, input_slug,
                 if not (s0 <= dtime <= s1):
                     continue  # Cull what's out of the interval
-                droundtime = round_time(dtime, 60*60)
+                droundtime = round_time(dtime, 60 * 60)
                 dkey = droundtime.strftime(precision)
                 x_hee = None
@@ -849,7 +860,7 @@ def get_data_for_target(target_config, input_slug,
 def generate_csv_contents(target_slug, input_slug, started_at, stopped_at):
     target_config = get_target_config(target_slug)
     log.debug("Crunching CSV contents for '%s'..." % target_config['name'])
-    si = StringIO.StringIO()
+    si = StringIO()
     cw = csv_writer(si)
     cw.writerow(PROPERTIES)
@@ -901,8 +912,9 @@ def generate_csv_file_if_needed(target_slug, input_slug,
             for trace in extract_tb(exc_traceback):
                 log.error(trace)
             if isfile(local_csv_file):
-                log.warn("Removing failed CSV '%s'..." % local_csv_file)
+                log.warning("Removing failed CSV '%s'..." % local_csv_file)
                 removefile(local_csv_file)
+            # pprint(e)
             abort(500, "Failed creating CSV '%s' : %s" % (filename, e))
@@ -1057,7 +1069,7 @@ def get_catalog_layers(input_slug, target_slug, started_at, stopped_at):
                         validates_any_constraint = True
                     for constraint in constraints:
                         validates_constraint = True
-                        for key, possible_values in constraint.iteritems():
+                        for key, possible_values in iter(constraint.items()):
                             actual_value = json_datum[_get_index_of_key(
                                 json_data, key
                             )]
@@ -1084,7 +1096,7 @@ def get_catalog_layers(input_slug, target_slug, started_at, stopped_at):
                     catalog_layers[config_layer['slug']].append({
                         'start': start_time.strftime(MOMENT_DATE_FMT),
-                        'stop':  stop_time.strftime(MOMENT_DATE_FMT),
+                        'stop': stop_time.strftime(MOMENT_DATE_FMT),
                     })
     return catalog_layers
@@ -1149,7 +1161,7 @@ def favicon():  # we want it served from the root, not from static/
 def home():
     increment_hit_counter()
     parameters = PARAMETERS.values()
-    parameters.sort(key=lambda x: x['position'])
+    parameters = sorted(parameters, key=lambda x: x['position'])
     input_slug = get_input_slug_from_query()
     targets = [t for t in config['targets'] if not t['locked']]
     started_at, stopped_at = get_interval_from_query()
@@ -1165,9 +1177,9 @@ def home():
         'started_at': started_at,
         'stopped_at': stopped_at,
         'planets': [s for s in config['targets'] if s['type'] == 'planet'],
-        'probes':  [s for s in config['targets'] if s['type'] == 'probe'],
-        'comets':  [s for s in config['targets'] if s['type'] == 'comet'],
-        'visits':  get_hit_counter(),
+        'probes': [s for s in config['targets'] if s['type'] == 'probe'],
+        'comets': [s for s in config['targets'] if s['type'] == 'comet'],
+        'visits': get_hit_counter(),
     })
@@ -1235,7 +1247,7 @@ def download_targets_tarball(targets, inp, started_at, stopped_at):
     """
     separator = '-'
     targets = targets.split(separator)
-    targets.sort()
+    targets.sorty()
     targets_configs = []
     for target in targets:
         if not target:
@@ -1307,7 +1319,7 @@ def download_targets_netcdf(targets, inp, params, started_at, stopped_at):
     """
     separator = '-'  # /!\ this char should never be in target's slugs
     targets = targets.split(separator)
-    targets.sort()
+    targets.sorty()
     targets_configs = []
     for target in targets:
         if not target:
@@ -1317,7 +1329,7 @@ def download_targets_netcdf(targets, inp, params, started_at, stopped_at):
         abort(400, "No valid targets specified. What are you doing?")
     params = params.split(separator)
-    params.sort()
+    params.sorty()
     if 0 == len(params):
         abort(400, "No valid parameters specified. What are you doing?")
     if not is_list_in_list(params, PARAMETERS.keys()):
@@ -1359,7 +1371,7 @@ def download_targets_netcdf(targets, inp, params, started_at, stopped_at):
                     started_at=started_at, stopped_at=stopped_at
                 )
                 dkeys = sorted(data)
-                dimension = 'dim_'+target_slug
+                dimension = 'dim_' + target_slug
                 nc_handle.createDimension(dimension, len(dkeys))
                 # TIME #
@@ -1432,7 +1444,7 @@ def download_targets_cdf(targets, inp, started_at, stopped_at):
     """
     separator = '-'  # /!\ this char should never be in target's slugs
     targets = targets.split(separator)
-    targets.sort()
+    targets.sorty()
     targets_configs = []
     for target in targets:
         if not target:
@@ -1552,7 +1564,7 @@ def download_targets_cdf(targets, inp, started_at, stopped_at):
                     else:
                         values_xhee.append(0)
                         values_yhee.append(0)
-                        log.warn("Orbit data for %s has NaNs." % target_slug)
+                        log.warning("Orbit data for %s has NaNs." % target_slug)
                 cdf_handle[kx] = values_xhee
                 cdf_handle[ky] = values_yhee
                 cdf_handle[kx].attrs['UNITS'] = 'Au'
@@ -1599,11 +1611,11 @@ def download_auroral_catalog_csv(target):
     header = ('time_min', 'time_max', 'thumbnail_url', 'external_link')
     if len(emissions):
         header = emissions[0].keys()
-    si = StringIO.StringIO()
+    si = StringIO()
     cw = csv_dict_writer(si, fieldnames=header)
     cw.writeheader()
     # 'time_min', 'time_max', 'thumbnail_url', 'external_link'
-    #cw.writerow(head)
+    # cw.writerow(head)
     log.debug("Writing auroral emissions CSV for %s..." % tc['name'])
     cw.writerows(emissions)
@@ -1617,8 +1629,6 @@ def download_auroral_catalog_csv(target):
     # return send_from_directory(CACHE_DIR, filename)
-
-
 @app.route("/test/auroral/<target>")
 def test_auroral_emissions(target):
     tc = validate_tap_target_config(target)
@@ -1666,7 +1676,7 @@ def cache_warmup():
     targets = get_active_targets()
     targets_slugs = [target['slug'] for target in targets]
-    targets_slugs.sort()
+    targets_slugs.sorty()
     update_spacepy()
     for target in targets:
@@ -1704,7 +1714,7 @@ def log_clear():
 #     cdf_to_inspect = get_path("../res/dummy.nc")
 #     cdf_to_inspect = get_path("../res/dummy_jupiter_coordinates.nc")
 #
-#     si = StringIO.StringIO()
+#     si = StringIO()
 #     cw = csv.DictWriter(si, fieldnames=['Name', 'Shape', 'Length'])
 #     cw.writeheader()
 #
@@ -272,7 +272,7 @@ var sw_configuration = {
       orbit: { a: {{ target.orbit.semimajor or 0 }}, b: {{ target.orbit.semiminor or 0 }} },
       img: '{{ static('img/target/'~target.slug~'_128.png') }}',
       layers: {
-{% for catalog_slug, catalog_intervals in target.catalog_layers.iteritems() %}
+{% for catalog_slug, catalog_intervals in target.catalog_layers.items() %}
         "{{ catalog_slug }}": [
 {% for interval in catalog_intervals %}
 {#        { start: "2018-03-28T00:00:00Z", stop: "2018-03-29T00:00:00Z" },   #}