From 9e44bb98e1cae80bcac0444f8dd53e3d60a6e919 Mon Sep 17 00:00:00 2001 From: Antoine Goutenoir Date: Sat, 30 Nov 2019 23:59:16 +0100 Subject: [PATCH] Support file uploads. --- content.yml | 36 +++++++++++++++++++++++++++++------- flaskr/controllers/main_controller.py | 83 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- flaskr/forms.py | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++---- flaskr/templates/estimate.html | 60 ++++++++++++++++++++++++++++++++++++++++++++++++------------ 4 files changed, 208 insertions(+), 26 deletions(-) diff --git a/content.yml b/content.yml index 651961b..2a1d77b 100644 --- a/content.yml +++ b/content.yml @@ -619,17 +619,27 @@ estimate: last_name: We will never share your data with anyone. origin_addresses: | Use en_US city and country names, without diacritics. -   +
The comma matters.
This is either a home city and a country or the cities and countries of the participants to the conference, meeting… + origin_addresses_file: | + If you provide a file, we'll use it instead of the list on the left. +
+ The spreadsheet's first sheet must have an Address column, + or a City and Country columns. destination_addresses: | This is either the cities and countries to travel to or the host city and country of the conference, meeting…
Please provide multiple cities and countries to compute the location of the minimum emission. + destination_addresses_file: | + If you provide a file, we'll use it instead of the list on the left. +
+ The spreadsheet's first sheet must have an Address column, + or a City and Country columns. # Labels accept HTML, but not markdown # Descriptions accept neither, since we use the HTML title attribute @@ -659,7 +669,7 @@ estimate: description: | One address per line, in the form `City, Country`. Make sure your addresses are correctly spelled. - # We MUST use the dumb CRLF pair for windows users + # We MUST use the dumb CRLF pair for dumb windows users placeholder: "Paris, France\r\nBerlin, Germany" # placeholder: | # Paris, France @@ -671,12 +681,24 @@ estimate: Make sure your addresses are correctly spelled. placeholder: | Washington, United States of America - compute_optimal_destination: - label: | - Compute the destination city that will minimize emissions
- (useful when setting up a meeting/conference) + origin_addresses_file: + label: Origin Cities description: | - We will only look through Cities specified in the Destination Cities. + Accepted files: CSV, XLS, XLSX. + We will use the Address column, or the City and Country columns. + error: Please use spreadsheet files only (CSV, XLS, XLSX) + destination_addresses_file: + label: Destination Cities + description: | + Accepted files: CSV, XLS, XLSX. + We will use the Address column, or the City and Country columns. + error: Please use spreadsheet files only (CSV, XLS, XLSX) +# compute_optimal_destination: +# label: | +# Compute the destination city that will minimize emissions
+# (useful when setting up a meeting/conference) +# description: | +# We will only look through Cities specified in the Destination Cities. use_atmosfair_rfi: label: | Use the RFI diff --git a/flaskr/controllers/main_controller.py b/flaskr/controllers/main_controller.py index 9e87db6..12bb00e 100644 --- a/flaskr/controllers/main_controller.py +++ b/flaskr/controllers/main_controller.py @@ -21,6 +21,9 @@ import csv # from io import StringIO from cStringIO import StringIO +import pandas +from pandas.compat import StringIO as PandasStringIO + main = Blueprint('main', __name__) @@ -56,6 +59,75 @@ def home(): ) +def gather_addresses(from_list, from_file): + addresses = [] + if from_file: + file_mimetype = from_file.mimetype + file_contents = from_file.read() + + rows_dicts = None + + if 'text/csv' == file_mimetype: + + rows_dicts = pandas \ + .read_csv(PandasStringIO(file_contents)) \ + .rename(str.lower, axis='columns') \ + .to_dict(orient="row") + + # Here are just *some* of the mimetypes that Microsoft's + # garbage spreadsheet files may have. + # application/vnd.ms-excel (official) + # application/msexcel + # application/x-msexcel + # application/x-ms-excel + # application/x-excel + # application/x-dos_ms_excel + # application/xls + # application/x-xls + # application/vnd.openxmlformats-officedocument.spreadsheetml.sheet + # ... Let's check extension instead. + + elif from_file.filename.endswith('xls') \ + or from_file.filename.endswith('xlsx'): + + rows_dicts = pandas \ + .read_excel(PandasStringIO(file_contents)) \ + .rename(str.lower, axis='columns') \ + .to_dict(orient="row") + + # Python 3.7 only + # elif from_file.filename.endswith('ods'): + # + # rows_dicts = read_ods(PandasStringIO(file_contents), 1) \ + # .rename(str.lower, axis='columns') \ + # .to_dict(orient="row") + + if rows_dicts is not None: + for row_dict in rows_dicts: + if 'address' in row_dict: + addresses.append(row_dict['address']) + continue + address = None + if 'city' in row_dict: + address = row_dict['city'] + if 'country' in row_dict: + if address is None: + address = row_dict['country'] + else: + address += "," + row_dict['country'] + if address is not None: + addresses.append(address) + else: + pass # what should we do here? raise? + else: + pass # what should we do here? raise? + + else: + addresses = from_list.replace("\r", '').split("\n") + + return "\n".join(addresses) + + @main.route("/estimate", methods=["GET", "POST"]) @main.route("/estimate.html", methods=["GET", "POST"]) def estimate(): @@ -72,10 +144,15 @@ def estimate(): estimation.last_name = form.last_name.data estimation.institution = form.institution.data estimation.status = StatusEnum.pending - estimation.origin_addresses = form.origin_addresses.data - estimation.destination_addresses = form.destination_addresses.data + estimation.origin_addresses = gather_addresses( + form.origin_addresses.data, + form.origin_addresses_file.data + ) + estimation.destination_addresses = gather_addresses( + form.destination_addresses.data, + form.destination_addresses_file.data + ) estimation.use_train_below_km = form.use_train_below_km.data - # estimation.compute_optimal_destination = form.compute_optimal_destination.data models_slugs = [] for model in models: if getattr(form, 'use_model_%s' % model.slug).data: diff --git a/flaskr/forms.py b/flaskr/forms.py index 058f7b9..e3f6fab 100644 --- a/flaskr/forms.py +++ b/flaskr/forms.py @@ -1,11 +1,14 @@ from flask_wtf import FlaskForm +from flask_wtf.file import FileAllowed from wtforms import \ StringField, \ PasswordField, \ TextAreaField, \ SelectField, \ - BooleanField + BooleanField, \ + FileField from wtforms import validators +from werkzeug.datastructures import FileStorage from .models import User from .content import content_dict as content @@ -98,6 +101,32 @@ class EstimateForm(FlaskForm): "placeholder": form_content['destination_addresses']['placeholder'] }, ) + origin_addresses_file = FileField( + label=form_content['origin_addresses_file']['label'], + description=form_content['origin_addresses_file']['description'], + validators=[ + # We disabled validators because they bug with multiple FileFields + # validators.Optional(), + # FileAllowed( + # ['csv', 'xls', 'xlsx'], + # form_content['origin_addresses_file']['error'] + # ) + ], + ) + destination_addresses_file = FileField( + label=form_content['destination_addresses_file']['label'], + description=form_content['destination_addresses_file']['description'], + validators=[ + # We disabled validators because they bug with multiple FileFields + # validators.Optional(), + # FileAllowed( + # ['csv', 'xls', 'xlsx'], + # form_content['destination_addresses_file']['error'] + # ) + ], + ) + + upload_set = ['csv', 'xls', 'xlsx'] # compute_optimal_destination = BooleanField( # label=form_content['compute_optimal_destination']['label'], @@ -132,11 +161,29 @@ class EstimateForm(FlaskForm): if not uses_at_least_one_model: last_model = getattr(self, 'use_model_%s' % models[-1].slug) - last_model.errors.append("Please select at least one model." - "  " # It's been a while - "What are you doing?") + last_model.errors.append( + "Please select at least one plane model, " + "even for train-only estimations." + ) return False + # Check uploaded files' extensions, if any + # We have to do this "by hand" because of a bug in flask wtf + if isinstance(self.origin_addresses_file.data, FileStorage): + fn = self.origin_addresses_file.data.filename.lower() + if fn and not any(fn.endswith('.' + x) for x in self.upload_set): + self.origin_addresses_file.errors.append( + form_content['origin_addresses_file']['error'] + ) + return False + if isinstance(self.destination_addresses_file.data, FileStorage): + fn = self.destination_addresses_file.data.filename.lower() + if fn and not any(fn.endswith('.' + x) for x in self.upload_set): + self.destination_addresses_file.errors.append( + form_content['destination_addresses_file']['error'] + ) + return False + return True diff --git a/flaskr/templates/estimate.html b/flaskr/templates/estimate.html index ec03218..4a8644a 100644 --- a/flaskr/templates/estimate.html +++ b/flaskr/templates/estimate.html @@ -14,7 +14,7 @@ {#############################################################################} -{# MACROS ######################################################################} +{# MACROS ####################################################################} {% macro render_field(field) %} @@ -60,7 +60,7 @@
-
+ {{ form.hidden_tag() }} {#
#} @@ -80,17 +80,53 @@
{{ render_field(form.institution) }}
-
- {{ render_field(form.origin_addresses) }} - - {{ content.estimate.help.origin_addresses | safe }} - + +
+
+
+ {{ render_field(form.origin_addresses) }} + + {{ content.estimate.help.origin_addresses | safe }} + +
+
+
+
+
+ OR +
+
+
+ {{ render_field(form.origin_addresses_file) }} + + {{ content.estimate.help.origin_addresses_file | safe }} + +
+
-
- {{ render_field(form.destination_addresses) }} - - {{ content.estimate.help.destination_addresses | safe }} - + +
+
+
+ {{ render_field(form.destination_addresses) }} + + {{ content.estimate.help.destination_addresses | safe }} + +
+
+
+
+
+ OR +
+
+
+ {{ render_field(form.destination_addresses_file) }} + + {{ content.estimate.help.destination_addresses_file | safe }} + +
+
{#
#} -- libgit2 0.21.2