Antoine Goutenoir · Antoine Goutenoir · Antoine Goutenoir · Antoine Goutenoir · Antoine Goutenoir · Antoine Goutenoir
Showing 15 changed files Show diff stats
Dockerfile
README.md
VERSION
docker-compose.yml
flaskr/__init__.py
flaskr/content.py
flaskr/controllers/main_controller.py
flaskr/forms.py
flaskr/geocoder.py
flaskr/models.py
flaskr/static/img/recap_scaling_laws.jpg
flaskr/templates/base.html
flaskr/templates/estimation-request.html
flaskr/templates/estimation.html
requirements.txt
-#FROM python:2.7
-FROM tiangolo/uwsgi-nginx-flask:python2.7
+FROM tiangolo/uwsgi-nginx-flask:python3.8
 ADD . /app
 WORKDIR /app
+RUN apt-get update && apt-get install -y cython3
 RUN pip install -r requirements.txt
 ENV FLASK_APP "flaskr"
 ENV FLASK_ENV "production"
@@ -2,7 +2,7 @@
 # Travel Carbon Footprint Calculator
 - https://travel-footprint-calculator.apps.goutenoir.com  (private demo)
-- http://travel-footprint-calculator.irap.omp.eu  (official, for later)
+- http://travel-footprint-calculator.irap.omp.eu  (official)
 ## Overview
@@ -16,20 +16,25 @@
 ## Installation
-Tested only on Python `2.7`.  _Sprint._ 
+Works on Python `2.7` up to `1.2` version.
+Works on Python `3.7` from `1.3` version.
+
 ### Create a virtual environment
 You don't _have to_.  But it's useful for development.
-    virtualenv venv
+    virtualenv .venv
 Then, source it to enable it.
-    source venv/bin/activate
+    source .venv/bin/activate
 ### Install the python dependencies
+You will need Cython.
+
+    sudo apt install cython
     pip install -r requirements.txt
 ### Create an empty database
@@ -44,18 +49,16 @@ Then, source it to enable it.
 ### Configure permissions
 `var/runs` must be writeable by the application.
+It is the file-based part of the database.
-
-## Build CSS and JS ()for prod)
+## Build CSS and JS (for prod)
     flask assets build
-
-
 ## Development
-    source venv/bin/activate
+    source .venv/bin/activate
     source .env.flaskrun
     flask run
-0.10.0
 \ No newline at end of file
+1.3.6
@@ -4,8 +4,8 @@
 version: '2.0'
 services:
-    com_goutenoir_apps_travel-footprint-calculator:
-        container_name: com_goutenoir_apps_travel-footprint-calculator
+    travel_footprint_calculator:
+        container_name: travel_footprint_calculator
         restart: always
         build: .
         volumes:
@@ -22,13 +22,12 @@ services:
             LETSENCRYPT_HOST: travel-footprint-calculator.apps.goutenoir.com
             LETSENCRYPT_EMAIL: antoine@goutenoir.com
             VIRTUAL_HOST: travel-footprint-calculator.apps.goutenoir.com
-            # 80 is the default port
             VIRTUAL_PORT: 80
             ###################################################
-    com_goutenoir_apps_travel-footprint-calculator_mahcron:
-        container_name: com_goutenoir_apps_travel-footprint-calculator_mahcron
+    travel_footprint_calculator_mahcron:
+        container_name: travel_footprint_calculator_mahcron
         image: jsonfry/curl-cron
         restart: always
         environment:
 #! ../venv/bin/python
 import os
-from markdown import markdown
 from dotenv import load_dotenv, find_dotenv, dotenv_values
+from markdown import markdown
+
 # Load config from .env ; do this before importing local libs (and flask)
 # 1. Write into OS environment -- if this fails you forgot to create .env file
 load_dotenv(find_dotenv(raise_error_if_not_found=True), override=True)
@@ -63,7 +64,7 @@ def create_app(object_name):
     app.config['BASIC_AUTH_USERNAME'] = os.getenv('ADMIN_USERNAME')
     app.config['BASIC_AUTH_PASSWORD'] = os.getenv('ADMIN_PASSWORD')
-    app.config['CAPTCHA_ENABLE'] = True
+    app.config['CAPTCHA_ENABLE'] = False
     app.config['CAPTCHA_LENGTH'] = 5
     app.config['CAPTCHA_WIDTH'] = 256
     app.config['CAPTCHA_HEIGHT'] = 158
 from collections import namedtuple
-from yaml import safe_load as yaml_safe_load
 from os.path import abspath, dirname, join
+from yaml import safe_load as yaml_safe_load
+
+# Move this to ENV, perhaps
+base_url = "https://travel-footprint-calculator.irap.omp.eu"
 PROJECT_DIRECTORY = abspath(dirname(dirname(__file__)))
@@ -21,7 +24,7 @@ class Struct(object):
     def __new__(cls, data):
         if isinstance(data, dict):
             return namedtuple(
-                'Struct', data.iterkeys()
+                'Struct', data.keys()
             )(
                 *(Struct(val) for val in data.values())
             )
@@ -33,9 +36,6 @@ class Struct(object):
 content = Struct(content_dict)
-# Move this to ENV, perhaps
-base_url = "https://travel-footprint-calculator.irap.omp.eu"
-
 # For Python3?
 # def dict2obj(d):
 import csv
 import re
-# from io import StringIO
-from cStringIO import StringIO
+# from cStringIO import StringIO
 from copy import deepcopy
-from os import unlink
+from io import StringIO
+from os import unlink, getenv
 from os.path import join
 import geopy
@@ -13,13 +13,14 @@ from flask import (
     Blueprint,
     Response,
     render_template,
+    request,
     flash,
     redirect,
     url_for,
     abort,
     send_from_directory,
 )
-from pandas.compat import StringIO as PandasStringIO
+# from pandas.compat import StringIO as PandasStringIO
 from wtforms import validators
 from yaml import safe_dump as yaml_dump
@@ -28,7 +29,7 @@ from flaskr.core import (
     get_emission_models,
     increment_hit_counter,
 )
-from flaskr.extensions import cache, send_email
+from flaskr.extensions import send_email
 from flaskr.forms import EstimateForm
 from flaskr.geocoder import CachedGeocoder
 from flaskr.models import db, Estimation, StatusEnum, ScenarioEnum
@@ -36,7 +37,7 @@ from flaskr.models import db, Estimation, StatusEnum, ScenarioEnum
 main = Blueprint('main', __name__)
-OUT_ENCODING = 'utf-8'
+#OUT_ENCODING = 'utf-8'
 # -----------------------------------------------------------------------------
@@ -48,7 +49,7 @@ pi_email = &quot;didier.barret@gmail.com&quot;  # todo: move to content YAML or .env
 @main.route('/favicon.ico')
-@cache.cached(timeout=10000)
+# @cache.cached(timeout=10000)
 def favicon():  # we want it served from the root, not from static/
     return send_from_directory(
         join(main.root_path, '..', 'static', 'img'),
@@ -82,14 +83,16 @@ def gather_addresses(from_list, from_file):
     addresses = []
     if from_file:
         file_mimetype = from_file.mimetype
-        file_contents = from_file.read()
+        file_contents = from_file.read().decode()
         rows_dicts = None
         if 'text/csv' == file_mimetype:
-
+            delimiter = ','
+            if ';' in file_contents:
+                delimiter = ';'
             rows_dicts = pandas \
-                .read_csv(PandasStringIO(file_contents)) \
+                .read_csv(StringIO(file_contents), delimiter=delimiter) \
                 .rename(str.lower, axis='columns') \
                 .to_dict(orient="row")
@@ -110,7 +113,7 @@ def gather_addresses(from_list, from_file):
                 or from_file.filename.endswith('xlsx'):
             rows_dicts = pandas \
-                .read_excel(PandasStringIO(file_contents)) \
+                .read_excel(StringIO(file_contents)) \
                 .rename(str.lower, axis='columns') \
                 .to_dict(orient="row")
@@ -153,8 +156,9 @@ def gather_addresses(from_list, from_file):
     for address in addresses:
         if not address:
             continue
-        if type(address).__name__ == 'str':
-            address = unicode(address, 'utf-8')
+        # if type(address).__name__ == 'str':
+        #     address = str(address).encode('utf-8')
+        address = str(address)
         if to_ignore.match(address) is not None:
             continue
         clean_addresses.append(address)
@@ -192,7 +196,7 @@ def estimate():  # register new estimation request, more accurately
                 form.origin_addresses_file.data
             )
         except validators.ValidationError as e:
-            form.origin_addresses_file.errors.append(e.message)
+            form.origin_addresses_file.errors.append(str(e))
             return show_form()
         try:
@@ -201,7 +205,7 @@ def estimate():  # register new estimation request, more accurately
                 form.destination_addresses_file.data
             )
         except validators.ValidationError as e:
-            form.destination_addresses_file.errors.append(e.message)
+            form.destination_addresses_file.errors.append(str(e))
             return show_form()
         estimation.use_train_below_km = form.use_train_below_km.data
@@ -372,7 +376,7 @@ def compute():  # process the queue of estimation requests
                 continue
             try:
-                origin = geocoder.geocode(origin_address.encode('utf-8'))
+                origin = geocoder.geocode(origin_address)
             except geopy.exc.GeopyError as e:
                 warning = u"Ignoring origin `%s` " \
                           u"since we failed to geocode it.\n%s\n" % (
@@ -428,9 +432,7 @@ def compute():  # process the queue of estimation requests
                 continue
             try:
-                destination = geocoder.geocode(
-                    destination_address.encode('utf-8')
-                )
+                destination = geocoder.geocode(destination_address)
             except geopy.exc.GeopyError as e:
                 warning = u"Ignoring destination `%s` " \
                           u"since we failed to geocode it.\n%s\n" % (
@@ -489,6 +491,13 @@ def compute():  # process the queue of estimation requests
         # UTILITY PRIVATE FUNCTION(S) #########################################
+        # _locations
+        def _get_location_key(_location):
+            return "%s, %s" % (
+                _get_city_key(_location),
+                _get_country_key(_location),
+            )
+
         def _get_city_key(_location):
             return _location.address.split(',')[0]
@@ -512,7 +521,7 @@ def compute():  # process the queue of estimation requests
             _results = {}
             footprints = {}
-            destinations_by_city_key = {}
+            destinations_by_key = {}
             cities_sum_foot = {}
             cities_sum_dist = {}
@@ -528,13 +537,13 @@ def compute():  # process the queue of estimation requests
                         extra_config=_extra_config,
                     )
-                    _key = _get_city_key(_destination)
-
-                    destinations_by_city_key[_key] = _destination
+                    _key = _get_location_key(_destination)
+                    destinations_by_key[_key] = _destination
                     if _key not in cities_dict:
                         cities_dict[_key] = {
-                            'city': _key,
+                            'location': _get_location_key(_destination),
+                            'city': _get_city_key(_destination),
                             'country': _get_country_key(_destination),
                             'address': _destination.address,
                             'latitude': _destination.latitude,
@@ -578,11 +587,12 @@ def compute():  # process the queue of estimation requests
                 city_train_trips = cities_dict_first_model[city]['train_trips']
                 city_plane_trips = cities_dict_first_model[city]['plane_trips']
                 cities_mean_dict[city] = {
-                    'city': city,
-                    'country': _get_country_key(destinations_by_city_key[city]),
-                    'address': destinations_by_city_key[city].address,
-                    'latitude': destinations_by_city_key[city].latitude,
-                    'longitude': destinations_by_city_key[city].longitude,
+                    'location': _get_location_key(destinations_by_key[city]),
+                    'city': _get_city_key(destinations_by_key[city]),
+                    'country': _get_country_key(destinations_by_key[city]),
+                    'address': destinations_by_key[city].address,
+                    'latitude': destinations_by_key[city].latitude,
+                    'longitude': destinations_by_key[city].longitude,
                     'footprint': city_mean_foot,
                     'distance': city_mean_dist,
                     'train_trips': city_train_trips,
@@ -638,19 +648,21 @@ def compute():  # process the queue of estimation requests
         # SCENARIO C : At Least One Origin, At Least One Destination ##########
         #
         # Run Scenario A for each Destination, and expose optimum Destination.
+        # Skip destinations already visited.  (collapse duplicate destinations)
         #
         else:
             estimation.scenario = ScenarioEnum.many_to_many
-            unique_city_keys = []
+            unique_location_keys = []
             result_cities = []
             for destination in destinations:
+                location_key = _get_location_key(destination)
                 city_key = _get_city_key(destination)
                 country_key = _get_country_key(destination)
-                if city_key in unique_city_keys:
+                if location_key in unique_location_keys:
                     continue
                 else:
-                    unique_city_keys.append(city_key)
+                    unique_location_keys.append(location_key)
                 city_results = compute_one_to_many(
                     _origin=destination,
@@ -659,6 +671,7 @@ def compute():  # process the queue of estimation requests
                 )
                 city_results['city'] = city_key
                 city_results['country'] = country_key
+                city_results['location'] = location_key
                 city_results['address'] = destination.address
                 city_results['latitude'] = destination.latitude
                 city_results['longitude'] = destination.longitude
@@ -699,7 +712,9 @@ def compute():  # process the queue of estimation requests
     except Exception as e:
         errmsg = u"Computation failed : %s" % (e,)
-        # errmsg = u"%s\n\n%s" % (errmsg, traceback.format_exc())
+        if 'production' != getenv('FLASK_ENV', 'production'):
+            import traceback
+            errmsg = u"%s\n\n%s" % (errmsg, traceback.format_exc())
         if estimation:
             _handle_failure(estimation, errmsg)
         return _respond(errmsg)
@@ -717,7 +732,7 @@ def consult_estimation(public_id, extension):
     except sqlalchemy.orm.exc.NoResultFound:
         return abort(404)
     except Exception as e:
-        # TODO: log?
+        # log? (or not)
         return abort(500)
     # allowed_formats = ['html']
@@ -761,6 +776,7 @@ def consult_estimation(public_id, extension):
         si = StringIO()
         cw = csv.writer(si, quoting=csv.QUOTE_ALL)
         cw.writerow([
+            u"location",
             u"city", u"country", u"address",
             u"latitude", u"longitude",
             u"co2_kg",
@@ -772,9 +788,11 @@ def consult_estimation(public_id, extension):
         results = estimation.get_output_dict()
         for city in results['cities']:
             cw.writerow([
-                city['city'].encode(OUT_ENCODING),
-                city['country'].encode(OUT_ENCODING),
-                city['address'].encode(OUT_ENCODING),
+                # city['location'].encode(OUT_ENCODING),
+                city['location'],
+                city['city'],
+                city['country'],
+                city['address'],
                 city.get('latitude', 0.0),
                 city.get('longitude', 0.0),
                 round(city['footprint'], 3),
@@ -948,6 +966,49 @@ def get_scaling_laws_csv():
     )
+@main.route('/geocode')
+@main.route('/geocode.html')
+def query_geocode():
+    requested = request.args.getlist('address')
+    if not requested:
+        requested = request.args.getlist('address[]')
+    if not requested:
+        requested = request.args.getlist('a')
+    if not requested:
+        requested = request.args.getlist('a[]')
+    # requested = _collect_request_args_list(('address', 'a'))
+    if not requested:
+        return Response(
+            response="""
+<p>
+Usage example: <a href="/geocode.html?address=Toulouse,France&amp;address=Paris,France">/geocode?address=Toulouse,France</a>
+</p>
+
+<p>
+Please do not request this endpoint more than every two seconds.
+</p>
+"""
+        )
+
+    response = u""
+
+    geocoder = CachedGeocoder()
+    for address in requested:
+        location = geocoder.geocode(address)
+
+        response += """
+<pre>
+Requested: `%s'
+Geocoded: `%s'
+Longitude: `%f`
+Latitude: `%f`
+Altitude: `%f` (unreliable)
+</pre>
+""" % (address, location, location.longitude, location.latitude, location.altitude)
+
+    return Response(response=response)
+
+
 @main.route("/test")
 # @basic_auth.required
 def dev_test():
@@ -11,7 +11,6 @@ from wtforms import validators
 from .content import content_dict as content
 from .core import models
-from .extensions import captcha
 from .models import User
 form_content = content['estimate']['form']
@@ -134,14 +133,14 @@ class EstimateForm(FlaskForm):
             # )
         ],
     )
-    captcha = StringField(
-        label=form_content['captcha']['label'],
-        description=form_content['captcha']['description'],
-        validators=[
-            validators.InputRequired(),
-            validators.Length(max=16),
-        ],
-    )
+    # captcha = StringField(
+    #     label=form_content['captcha']['label'],
+    #     description=form_content['captcha']['description'],
+    #     validators=[
+    #         validators.InputRequired(),
+    #         validators.Length(max=16),
+    #     ],
+    # )
     upload_set = ['csv', 'xls', 'xlsx']
@@ -169,12 +168,12 @@ class EstimateForm(FlaskForm):
         if not check_validate:
             return False
-        if self.captcha.data != IS_HUMAN:
-            if not captcha.validate():
-                self.captcha.errors.append(
-                    "Captcha do not match.  Try again."
-                )
-                return False
+        # if self.captcha.data != IS_HUMAN:
+        #     if not captcha.validate():
+        #         self.captcha.errors.append(
+        #             "Captcha do not match.  Try again."
+        #         )
+        #         return False
         # Origins must be set either by field or file
         if (
-import geopy
 import shelve
+import ssl
 import time
-from core import get_path
+import certifi
+import geopy
+import geopy.geocoders
+
+from flaskr.core import get_path
+
+ssl_ctx = ssl.create_default_context(cafile=certifi.where())
+geopy.geocoders.options.default_ssl_context = ssl_ctx
 class CachedGeocoder:
     def __init__(self, source="Nominatim", geocache="geocache.db"):
-        self.geocoder = getattr(geopy.geocoders, source)()
+        self.geocoder = getattr(geopy.geocoders, source)(scheme='https')
         self.cache = shelve.open(get_path(geocache), writeback=True)
         # self.timestamp = time.time() + 1.5
@@ -25,5 +32,8 @@ class CachedGeocoder:
             )
         return self.cache[address]
+    def __del__(self):
+        self.close()
+
     def close(self):
         self.cache.close()
@@ -8,7 +8,7 @@ from flask_sqlalchemy import SQLAlchemy
 from werkzeug.security import generate_password_hash, check_password_hash
 from yaml import safe_load as yaml_load
-from content import get_path, base_url
+from flaskr.content import get_path, base_url
 from flaskr.core import generate_unique_id, models
 # These are not the emission "models" in the scientific meaning of the word.
@@ -70,6 +70,43 @@
                 {% endif %}
             {% endwith %}
+            <style>
+                .nojs {
+                    opacity: 0;
+                    animation: 16.18s fadeIn;
+                    animation-delay: 1.618s;
+                    animation-fill-mode: forwards;
+
+                    visibility: hidden;
+                    pointer-events: none;
+                }
+
+                @keyframes fadeIn {
+                    0% {
+                        opacity: 0;
+                    }
+                    25% {
+                        visibility: visible;
+                        opacity: 1;
+                    }
+                    93% {
+                        visibility: visible;
+                        opacity: 1;
+                    }
+                    99% {
+                        visibility: visible;
+                        opacity: 0;
+                    }
+                    100% {
+                        visibility: hidden;
+                    }
+                }
+            </style>
+            <div class="alert alert-danger nojs">
+                Javascript appears disabled, and we totally get that.
+                You'll need javascript to render the plots, though.
+            </div>
+
             {% block body %}
             {% endblock %}
         </div>
@@ -176,30 +176,30 @@
                 <small class="form-text text-muted">{{ content.estimate.help.run_name | safe }}</small>
             </div>
-            <div class="form-group">
-                <dt>
-                    {{ form.captcha.label }}
-                    <span class="required-asterisk" title="This field is required.">*</span>
-                </dt>
-                <dd>
-                    {{ captcha() }}
-                    <br><br>
-
-                    {{ form.captcha(
-                        title=form.captcha.description,
-                        class_="form-control",
-                        placeholder="Please write the numbers you see in the image above."
-                    ) | safe }}
-
-                    {% if form.captcha.errors -%}
-                    <ul class="errors text-danger has-error">
-                    {% for error in form.captcha.errors %}
-                        <li>{{ error }}</li>
-                    {% endfor %}
-                    </ul>
-                    {%- endif %}
-                </dd>
-            </div>
+{#            <div class="form-group">#}
+{#                <dt>#}
+{#                    {{ form.captcha.label }}#}
+{#                    <span class="required-asterisk" title="This field is required.">*</span>#}
+{#                </dt>#}
+{#                <dd>#}
+{#                    {{ captcha() }}#}
+{#                    <br><br>#}
+{##}
+{#                    {{ form.captcha(#}
+{#                        title=form.captcha.description,#}
+{#                        class_="form-control",#}
+{#                        placeholder="Please write the numbers you see in the image above."#}
+{#                    ) | safe }}#}
+{##}
+{#                    {% if form.captcha.errors -%}#}
+{#                    <ul class="errors text-danger has-error">#}
+{#                    {% for error in form.captcha.errors %}#}
+{#                        <li>{{ error }}</li>#}
+{#                    {% endfor %}#}
+{#                    </ul>#}
+{#                    {%- endif %}#}
+{#                </dd>#}
+{#            </div>#}
             <button type="submit" class="btn btn-primary">
                 Submit a Request
@@ -122,7 +122,7 @@
 {% endif %}
 {#############################################################################}
-{# SORTED EMISSIONS INEQUALITY ##########################################}
+{# SORTED EMISSIONS INEQUALITY ###############################################}
 {# That plot makes no sense with our many to many data. #}
 {% if not estimation.is_many_to_many() %}
 <div class="row">
@@ -378,13 +378,12 @@ jQuery(document).ready(function($){
     console.info("[Footprint Lollipop] Starting…");
     var vizid = "#cities_footprints_d3viz_lollipop";
     var csvUrl = "/estimation/{{ estimation.public_id }}.csv";
-    var y_key = 'city';
+    var y_key = 'address';
     var x_key = 'co2_kg';
-    var margin = {top: 40, right: 40, bottom: 150, left: 180},
-        height = Math.max(300, 100 + 16*plots_config['cities_count']) - margin.top - margin.bottom;
-    var width = Math.max(800, $(vizid).parent().width());
-    width = width - margin.left - margin.right;
+    var margin = {top: 40, right: 40, bottom: 150, left: 180};
+    var height = Math.max(300, 100 + 16*plots_config['cities_count']) - margin.top - margin.bottom;
+    var width = Math.max(800, $(vizid).parent().width()) - margin.left - margin.right;
     var svg_tag = d3.select(vizid)
       .append("svg")
@@ -404,6 +403,20 @@ jQuery(document).ready(function($){
     d3.csv(csvUrl).then(function (data) {
         console.info("[Footprint Lollipop] Generating…");
+
+        // Resize left margin from locations' character length
+        var max_character_length = 0;
+        data.forEach(function(datum){
+            max_character_length = Math.max(max_character_length, datum[y_key].length);
+        });
+        margin.left = Math.min(Math.round(0.618 * width), 42 + Math.round(5.13*max_character_length));
+        width = Math.max(800, $(vizid).parent().width()) - margin.left - margin.right;
+        svg_tag
+            .attr("width", width + margin.left + margin.right)
+            .attr("height", height + margin.top + margin.bottom);
+        svg.attr("transform",
+              "translate(" + margin.left + "," + margin.top + ")");
+
         // Extrema
         var data_x_max = d3.max(data, function (d) {
             return parseFloat(d[x_key]);
@@ -2,7 +2,7 @@
 Flask==1.1.1
 # Flask Extensions
-Flask-Admin==1.5.4
+Flask-Admin==1.5.7
 Flask-Assets==0.12
 Flask-BasicAuth==0.2.0
 Flask-Caching==1.7.2
@@ -10,7 +10,7 @@ Flask-DebugToolbar==0.10.0
 Flask-Login==0.4.0
 Flask-Mail==0.9.1
 Flask-Script==2.0.5
-Flask-SQLAlchemy==2.1
+Flask-SQLAlchemy==2.5.1
 Flask-WTF==0.14
 Flask-Sessionstore==0.4.5
 Flask-Session-Captcha==1.2.0
@@ -21,9 +21,10 @@ cssmin==0.2.0
 jsmin==2.2.1
 pyyaml==5.1.2
 Markdown==3.1.1
-numpy==1.16.5
+numpy==1.18.5
 enum34==1.1.6
-geopy==1.20.0
+geopy==1.23.0
+certifi==2019.11.28
 python-dotenv==0.10.3
 # Force stable werkzeug
@@ -33,7 +34,7 @@ Werkzeug==0.16.1
 # Spreadsheet reading
 # Note that 0.24 is the most recent version still supporting python 2.7
-pandas==0.24.2
+pandas==0.25.3
 # Excel support for wisdom-impaired Microsoft users
 xlrd==1.2.0
 # ODS reading support is only available natively in pandas 0.25