Support file uploads.

Antoine Goutenoir
1 parent 56c7b457
Showing 4 changed files with 208 additions and 26 deletions Show diff stats
content.yml
flaskr/controllers/main_controller.py
flaskr/forms.py
flaskr/templates/estimate.html
@@ -619,17 +619,27 @@ estimate:
     last_name: We will never share your data with anyone.
     origin_addresses: |
       Use <code>en_US</code> city and country names, without diacritics.
-      &nbsp;
+      <br>
       The comma matters.
       <br>
       This is either a home city and a country
       or the cities and countries of the participants to the conference, meeting…
+    origin_addresses_file: |
+      If you provide a file, we'll use it instead of the list on the left.
+      <br>
+      The spreadsheet's first sheet must have an <code>Address</code> column,
+      or a <code>City</code> and <code>Country</code> columns.
     destination_addresses: |
       This is either the cities and countries to travel to
       or the host city and country of the conference, meeting…
       <br>
       Please provide multiple cities and countries to compute the location
       of the minimum emission.
+    destination_addresses_file: |
+      If you provide a file, we'll use it instead of the list on the left.
+      <br>
+      The spreadsheet's first sheet must have an <code>Address</code> column,
+      or a <code>City</code> and <code>Country</code> columns.
  
   # Labels accept HTML, but not markdown
   # Descriptions accept neither, since we use the HTML title attribute
@@ -659,7 +669,7 @@ estimate:
       description: |
         One address per line, in the form `City, Country`.
         Make sure your addresses are correctly spelled.
-      # We MUST use the dumb CRLF pair for windows users
+      # We MUST use the dumb CRLF pair for dumb windows users
       placeholder: "Paris, France\r\nBerlin, Germany"
 #      placeholder: |
 #        Paris, France
@@ -671,12 +681,24 @@ estimate:
         Make sure your addresses are correctly spelled.
       placeholder: |
         Washington, United States of America
-    compute_optimal_destination:
-      label: |
-        Compute the destination city that will minimize emissions <br>
-        (useful when setting up a meeting/conference)
+    origin_addresses_file:
+      label: Origin Cities
       description: |
-        We will only look through Cities specified in the Destination Cities.
+        Accepted files: CSV, XLS, XLSX.
+        We will use the Address column, or the City and Country columns.
+      error: Please use spreadsheet files only (CSV, XLS, XLSX)
+    destination_addresses_file:
+      label: Destination Cities
+      description: |
+        Accepted files: CSV, XLS, XLSX.
+        We will use the Address column, or the City and Country columns.
+      error: Please use spreadsheet files only (CSV, XLS, XLSX)
+#    compute_optimal_destination:
+#      label: |
+#        Compute the destination city that will minimize emissions <br>
+#        (useful when setting up a meeting/conference)
+#      description: |
+#        We will only look through Cities specified in the Destination Cities.
     use_atmosfair_rfi:
       label: |
         Use the <acronym title="Radiative Forcing Index">RFI</acronym>
@@ -21,6 +21,9 @@ import csv
 # from io import StringIO
 from cStringIO import StringIO
  
+import pandas
+from pandas.compat import StringIO as PandasStringIO
+
 main = Blueprint('main', __name__)
  
  
@@ -56,6 +59,75 @@ def home():
     )
  
  
+def gather_addresses(from_list, from_file):
+    addresses = []
+    if from_file:
+        file_mimetype = from_file.mimetype
+        file_contents = from_file.read()
+
+        rows_dicts = None
+
+        if 'text/csv' == file_mimetype:
+
+            rows_dicts = pandas \
+                .read_csv(PandasStringIO(file_contents)) \
+                .rename(str.lower, axis='columns') \
+                .to_dict(orient="row")
+
+        # Here are just *some* of the mimetypes that Microsoft's
+        # garbage spreadsheet files may have.
+        # application/vnd.ms-excel (official)
+        # application/msexcel
+        # application/x-msexcel
+        # application/x-ms-excel
+        # application/x-excel
+        # application/x-dos_ms_excel
+        # application/xls
+        # application/x-xls
+        # application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
+        # ... Let's check extension instead.
+
+        elif from_file.filename.endswith('xls') \
+                or from_file.filename.endswith('xlsx'):
+
+            rows_dicts = pandas \
+                .read_excel(PandasStringIO(file_contents)) \
+                .rename(str.lower, axis='columns') \
+                .to_dict(orient="row")
+
+        # Python 3.7 only
+        # elif from_file.filename.endswith('ods'):
+        #
+        #     rows_dicts = read_ods(PandasStringIO(file_contents), 1) \
+        #         .rename(str.lower, axis='columns') \
+        #         .to_dict(orient="row")
+
+        if rows_dicts is not None:
+            for row_dict in rows_dicts:
+                if 'address' in row_dict:
+                    addresses.append(row_dict['address'])
+                    continue
+                address = None
+                if 'city' in row_dict:
+                    address = row_dict['city']
+                if 'country' in row_dict:
+                    if address is None:
+                        address = row_dict['country']
+                    else:
+                        address += "," + row_dict['country']
+                if address is not None:
+                    addresses.append(address)
+                else:
+                    pass  # what should we do here? raise?
+        else:
+            pass # what should we do here? raise?
+
+    else:
+        addresses = from_list.replace("\r", '').split("\n")
+
+    return "\n".join(addresses)
+
+
 @main.route("/estimate", methods=["GET", "POST"])
 @main.route("/estimate.html", methods=["GET", "POST"])
 def estimate():
@@ -72,10 +144,15 @@ def estimate():
         estimation.last_name = form.last_name.data
         estimation.institution = form.institution.data
         estimation.status = StatusEnum.pending
-        estimation.origin_addresses = form.origin_addresses.data
-        estimation.destination_addresses = form.destination_addresses.data
+        estimation.origin_addresses = gather_addresses(
+            form.origin_addresses.data,
+            form.origin_addresses_file.data
+        )
+        estimation.destination_addresses = gather_addresses(
+            form.destination_addresses.data,
+            form.destination_addresses_file.data
+        )
         estimation.use_train_below_km = form.use_train_below_km.data
-        # estimation.compute_optimal_destination = form.compute_optimal_destination.data
         models_slugs = []
         for model in models:
             if getattr(form, 'use_model_%s' % model.slug).data:
 from flask_wtf import FlaskForm
+from flask_wtf.file import FileAllowed
 from wtforms import \
     StringField, \
     PasswordField, \
     TextAreaField, \
     SelectField, \
-    BooleanField
+    BooleanField, \
+    FileField
 from wtforms import validators
+from werkzeug.datastructures import FileStorage
  
 from .models import User
 from .content import content_dict as content
@@ -98,6 +101,32 @@ class EstimateForm(FlaskForm):
             "placeholder": form_content['destination_addresses']['placeholder']
         },
     )
+    origin_addresses_file = FileField(
+        label=form_content['origin_addresses_file']['label'],
+        description=form_content['origin_addresses_file']['description'],
+        validators=[
+            # We disabled validators because they bug with multiple FileFields
+            # validators.Optional(),
+            # FileAllowed(
+            #     ['csv', 'xls', 'xlsx'],
+            #     form_content['origin_addresses_file']['error']
+            # )
+        ],
+    )
+    destination_addresses_file = FileField(
+        label=form_content['destination_addresses_file']['label'],
+        description=form_content['destination_addresses_file']['description'],
+        validators=[
+            # We disabled validators because they bug with multiple FileFields
+            # validators.Optional(),
+            # FileAllowed(
+            #     ['csv', 'xls', 'xlsx'],
+            #     form_content['destination_addresses_file']['error']
+            # )
+        ],
+    )
+
+    upload_set = ['csv', 'xls', 'xlsx']
  
     # compute_optimal_destination = BooleanField(
     #     label=form_content['compute_optimal_destination']['label'],
@@ -132,11 +161,29 @@ class EstimateForm(FlaskForm):
  
         if not uses_at_least_one_model:
             last_model = getattr(self, 'use_model_%s' % models[-1].slug)
-            last_model.errors.append("Please select at least one model."
-                                      "&nbsp;&nbsp;"  # It's been a while
-                                      "<em>What are you doing?</em>")
+            last_model.errors.append(
+                "Please select at least one plane model, "
+                "<em>even for train-only estimations.</em>"
+            )
             return False
  
+        # Check uploaded files' extensions, if any
+        # We have to do this "by hand" because of a bug in flask wtf
+        if isinstance(self.origin_addresses_file.data, FileStorage):
+            fn = self.origin_addresses_file.data.filename.lower()
+            if fn and not any(fn.endswith('.' + x) for x in self.upload_set):
+                self.origin_addresses_file.errors.append(
+                    form_content['origin_addresses_file']['error']
+                )
+                return False
+        if isinstance(self.destination_addresses_file.data, FileStorage):
+            fn = self.destination_addresses_file.data.filename.lower()
+            if fn and not any(fn.endswith('.' + x) for x in self.upload_set):
+                self.destination_addresses_file.errors.append(
+                    form_content['destination_addresses_file']['error']
+                )
+                return False
+
         return True
  
  
@@ -14,7 +14,7 @@
  
  
 {#############################################################################}
-{# MACROS ######################################################################}
+{# MACROS ####################################################################}
  
  
 {% macro render_field(field) %}
@@ -60,7 +60,7 @@
 <div class="row">
     <div class="col-md-2"></div>
     <div class="col-md-8">
-        <form role="form" action="{{ url_for('.estimate') }}" method="post">
+        <form role="form" action="{{ url_for('.estimate') }}" method="post" enctype="multipart/form-data">
             {{ form.hidden_tag() }}
  
 {#            <div class="form-group">#}
@@ -80,17 +80,53 @@
             <div class="form-group">
                 {{ render_field(form.institution) }}
             </div>
-            <div class="form-group">
-                {{ render_field(form.origin_addresses) }}
-                <small class="form-text text-muted">
-                    {{ content.estimate.help.origin_addresses | safe }}
-                </small>
+
+            <div class="row">
+                <div class="col-7">
+                    <div class="form-group">
+                        {{ render_field(form.origin_addresses) }}
+                        <small class="form-text text-muted">
+                            {{ content.estimate.help.origin_addresses | safe }}
+                        </small>
+                    </div>
+                </div>
+                <div class="col-1">
+                    <br>
+                    <br>
+                    OR
+                </div>
+                <div class="col-4">
+                    <div class="form-group">
+                        {{ render_field(form.origin_addresses_file) }}
+                        <small class="form-text text-muted">
+                            {{ content.estimate.help.origin_addresses_file | safe }}
+                        </small>
+                    </div>
+                </div>
             </div>
-            <div class="form-group">
-                {{ render_field(form.destination_addresses) }}
-                <small class="form-text text-muted">
-                    {{ content.estimate.help.destination_addresses | safe }}
-                </small>
+
+            <div class="row">
+                <div class="col-7">
+                    <div class="form-group">
+                        {{ render_field(form.destination_addresses) }}
+                        <small class="form-text text-muted">
+                            {{ content.estimate.help.destination_addresses | safe }}
+                        </small>
+                    </div>
+                </div>
+                <div class="col-1">
+                    <br>
+                    <br>
+                    OR
+                </div>
+                <div class="col-4">
+                    <div class="form-group">
+                        {{ render_field(form.destination_addresses_file) }}
+                        <small class="form-text text-muted">
+                            {{ content.estimate.help.destination_addresses_file | safe }}
+                        </small>
+                    </div>
+                </div>
             </div>
  
 {#            <div class="form-check form-group">#}