Blame view

flaskr/controllers/main_controller.py 23.3 KB
3ac083cb   Antoine Goutenoir   Add more details ...
1
import traceback
b935618e   Antoine Goutenoir   Count the number ...
2
3
from copy import deepcopy

314c65e2   Antoine Goutenoir   Implement Scenari...
4
5
import geopy
import sqlalchemy
51f564d3   Antoine Goutenoir   Add a big chunk o...
6

a4c03d8e   Antoine Goutenoir   Add the controlle...
7
from flask import Blueprint, render_template, flash, request, redirect, \
67f85bce   Antoine Goutenoir   Generate a CSV fi...
8
    url_for, abort, send_from_directory, Response
461850db   Antoine Goutenoir   Yet another joyfu...
9
from os.path import join
7b4d2926   Goutte   Rework the contro...
10

8ae021a2   Antoine Goutenoir   Merge shelved cha...
11
12
from os import unlink

b9fc86c3   Antoine Goutenoir   Secure the admin ...
13
from flaskr.extensions import cache, basic_auth
7b4d2926   Goutte   Rework the contro...
14
from flaskr.forms import LoginForm, EstimateForm
a1f12452   Antoine Goutenoir   Add the new conte...
15
from flaskr.models import db, User, Estimation, StatusEnum, ScenarioEnum
24f55cde   Antoine Goutenoir   Geocode destinati...
16
from flaskr.geocoder import CachedGeocoder
7b4d2926   Goutte   Rework the contro...
17

78eb2a62   Antoine Goutenoir   Use the counter.
18
19
from flaskr.core import generate_unique_id, \
    get_emission_models, increment_hit_counter
51f564d3   Antoine Goutenoir   Add a big chunk o...
20
from flaskr.content import content
7b4d2926   Goutte   Rework the contro...
21

f2fbbb72   Antoine Goutenoir   Display a nice er...
22
23
from wtforms import validators

70aa301f   Antoine Goutenoir   Implement another...
24
25
from yaml import safe_dump as yaml_dump

67f85bce   Antoine Goutenoir   Generate a CSV fi...
26
27
28
import csv
# from io import StringIO
from cStringIO import StringIO
24f55cde   Antoine Goutenoir   Geocode destinati...
29

9e44bb98   Antoine Goutenoir   Support file uplo...
30
31
32
import pandas
from pandas.compat import StringIO as PandasStringIO

7b4d2926   Goutte   Rework the contro...
33
34
35
main = Blueprint('main', __name__)


9621b8a5   Antoine Goutenoir   Fix the CSV gener...
36
37
38
OUT_ENCODING = 'utf-8'


67f85bce   Antoine Goutenoir   Generate a CSV fi...
39
# -----------------------------------------------------------------------------
78eb2a62   Antoine Goutenoir   Use the counter.
40
# -----------------------------------------------------------------------------
67f85bce   Antoine Goutenoir   Generate a CSV fi...
41
42


461850db   Antoine Goutenoir   Yet another joyfu...
43
@main.route('/favicon.ico')
7f7c6b10   Antoine Goutenoir   Disable RFI in th...
44
@cache.cached(timeout=10000)
461850db   Antoine Goutenoir   Yet another joyfu...
45
46
47
48
49
50
51
def favicon():  # we want it served from the root, not from static/
    return send_from_directory(
        join(main.root_path, '..', 'static', 'img'),
        'favicon.ico', mimetype='image/vnd.microsoft.icon'
    )


7b4d2926   Goutte   Rework the contro...
52
@main.route('/')
b0ffb1ba   Antoine Goutenoir   Review actively.
53
54
@main.route('/home')
@main.route('/home.html')
38375935   Antoine Goutenoir   Remove the cache ...
55
# @cache.cached(timeout=1000)
7b4d2926   Goutte   Rework the contro...
56
def home():
4c862b54   Antoine Goutenoir   Add a grouped bar...
57
    models = get_emission_models()
a3e9d0fc   Antoine Goutenoir   Fix home plot leg...
58
59
60
    models_dict = {}
    for model in models:
        models_dict[model.slug] = model.__dict__
78eb2a62   Antoine Goutenoir   Use the counter.
61
    increment_hit_counter()
4c862b54   Antoine Goutenoir   Add a grouped bar...
62
63
    return render_template(
        'home.html',
a3e9d0fc   Antoine Goutenoir   Fix home plot leg...
64
        models=models_dict,
4c862b54   Antoine Goutenoir   Add a grouped bar...
65
        colors=[model.color for model in models],
a3e9d0fc   Antoine Goutenoir   Fix home plot leg...
66
        labels=[model.name for model in models],
4c862b54   Antoine Goutenoir   Add a grouped bar...
67
    )
7b4d2926   Goutte   Rework the contro...
68
69


9e44bb98   Antoine Goutenoir   Support file uplo...
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
def gather_addresses(from_list, from_file):
    addresses = []
    if from_file:
        file_mimetype = from_file.mimetype
        file_contents = from_file.read()

        rows_dicts = None

        if 'text/csv' == file_mimetype:

            rows_dicts = pandas \
                .read_csv(PandasStringIO(file_contents)) \
                .rename(str.lower, axis='columns') \
                .to_dict(orient="row")

        # Here are just *some* of the mimetypes that Microsoft's
        # garbage spreadsheet files may have.
        # application/vnd.ms-excel (official)
        # application/msexcel
        # application/x-msexcel
        # application/x-ms-excel
        # application/x-excel
        # application/x-dos_ms_excel
        # application/xls
        # application/x-xls
        # application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
        # ... Let's check extension instead.

        elif from_file.filename.endswith('xls') \
                or from_file.filename.endswith('xlsx'):

            rows_dicts = pandas \
                .read_excel(PandasStringIO(file_contents)) \
                .rename(str.lower, axis='columns') \
                .to_dict(orient="row")

        # Python 3.7 only
        # elif from_file.filename.endswith('ods'):
        #
        #     rows_dicts = read_ods(PandasStringIO(file_contents), 1) \
        #         .rename(str.lower, axis='columns') \
        #         .to_dict(orient="row")

        if rows_dicts is not None:
            for row_dict in rows_dicts:
                if 'address' in row_dict:
                    addresses.append(row_dict['address'])
                    continue
                address = None
                if 'city' in row_dict:
                    address = row_dict['city']
                if 'country' in row_dict:
                    if address is None:
                        address = row_dict['country']
                    else:
                        address += "," + row_dict['country']
                if address is not None:
                    addresses.append(address)
                else:
f2fbbb72   Antoine Goutenoir   Display a nice er...
129
130
131
                    raise validators.ValidationError(
                        "We could not find Address data in the spreadsheet."
                    )
9e44bb98   Antoine Goutenoir   Support file uplo...
132
        else:
f2fbbb72   Antoine Goutenoir   Display a nice er...
133
134
135
            raise validators.ValidationError(
                "We could not find any data in the spreadsheet."
            )
9e44bb98   Antoine Goutenoir   Support file uplo...
136
137
138
139

    else:
        addresses = from_list.replace("\r", '').split("\n")

68da7cd4   Antoine Goutenoir   Improve robustnes...
140
141
142
143
144
145
146
147
148
149
150
151
    clean_addresses = []
    for address in addresses:
        if not address:
            continue
        elif type(address).__name__ == 'str':
            clean_addresses.append(unicode(address, 'utf-8'))
        else:
            clean_addresses.append(address)
    addresses = clean_addresses

    # Remove empty lines (if any) and white characters
    addresses = [a.strip() for a in addresses if a]
8ec0ce68   Antoine Goutenoir   Improve robustnes...
152

9e44bb98   Antoine Goutenoir   Support file uplo...
153
154
155
    return "\n".join(addresses)


7b4d2926   Goutte   Rework the contro...
156
@main.route("/estimate", methods=["GET", "POST"])
b0ffb1ba   Antoine Goutenoir   Review actively.
157
@main.route("/estimate.html", methods=["GET", "POST"])
7b4d2926   Goutte   Rework the contro...
158
def estimate():
77e86148   Antoine Goutenoir   Fake support for ...
159
    models = get_emission_models()
7b4d2926   Goutte   Rework the contro...
160
161
    form = EstimateForm()

f2fbbb72   Antoine Goutenoir   Display a nice er...
162
163
164
    def show_form():
        return render_template("estimate.html", form=form, models=models)

7b4d2926   Goutte   Rework the contro...
165
166
    if form.validate_on_submit():

7b4d2926   Goutte   Rework the contro...
167
        estimation = Estimation()
a4c03d8e   Antoine Goutenoir   Add the controlle...
168
        # estimation.email = form.email.data
8ae021a2   Antoine Goutenoir   Merge shelved cha...
169
        estimation.run_name = form.run_name.data
7b4d2926   Goutte   Rework the contro...
170
171
        estimation.first_name = form.first_name.data
        estimation.last_name = form.last_name.data
16f69d07   Antoine Goutenoir   Add an (unsecured...
172
        estimation.institution = form.institution.data
24f55cde   Antoine Goutenoir   Geocode destinati...
173
        estimation.status = StatusEnum.pending
f2fbbb72   Antoine Goutenoir   Display a nice er...
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192

        try:
            estimation.origin_addresses = gather_addresses(
                form.origin_addresses.data,
                form.origin_addresses_file.data
            )
        except validators.ValidationError as e:
            form.origin_addresses_file.errors.append(e.message)
            return show_form()

        try:
            estimation.destination_addresses = gather_addresses(
                form.destination_addresses.data,
                form.destination_addresses_file.data
            )
        except validators.ValidationError as e:
            form.destination_addresses_file.errors.append(e.message)
            return show_form()

4276f1aa   Antoine Goutenoir   Support train emi...
193
        estimation.use_train_below_km = form.use_train_below_km.data
f2fbbb72   Antoine Goutenoir   Display a nice er...
194

77e86148   Antoine Goutenoir   Fake support for ...
195
196
        models_slugs = []
        for model in models:
35fbac1f   Antoine Goutenoir   Fix a blooper.
197
            if getattr(form, 'use_model_%s' % model.slug).data:
77e86148   Antoine Goutenoir   Fake support for ...
198
199
                models_slugs.append(model.slug)
        estimation.models_slugs = "\n".join(models_slugs)
7b4d2926   Goutte   Rework the contro...
200
201
202
203
204

        db.session.add(estimation)
        db.session.commit()

        flash("Estimation request submitted successfully.", "success")
a4c03d8e   Antoine Goutenoir   Add the controlle...
205
206
207
        return redirect(url_for(
            endpoint=".consult_estimation",
            public_id=estimation.public_id,
91751451   Antoine Goutenoir   Shift the API to ...
208
            extension='html'
a4c03d8e   Antoine Goutenoir   Add the controlle...
209
        ))
7b4d2926   Goutte   Rework the contro...
210
211
        # return render_template("estimate-debrief.html", form=form)

f2fbbb72   Antoine Goutenoir   Display a nice er...
212
    return show_form()
7b4d2926   Goutte   Rework the contro...
213

4392f295   Goutte   Update the Estima...
214

15e57dca   Antoine Goutenoir   Naming things and...
215
@main.route("/invalidate")
4276f1aa   Antoine Goutenoir   Support train emi...
216
@main.route("/invalidate.html")
15e57dca   Antoine Goutenoir   Naming things and...
217
218
219
220
221
222
223
224
225
226
def invalidate():
    stuck_estimations = Estimation.query \
        .filter_by(status=StatusEnum.working) \
        .all()

    for estimation in stuck_estimations:
        estimation.status = StatusEnum.failure
        estimation.errors = "Invalidated. Try again."
        db.session.commit()

8ae021a2   Antoine Goutenoir   Merge shelved cha...
227
228
229
230
231
232
233
234
235
236
237
    return "Estimations invalidated: %d" % len(stuck_estimations)


@main.route("/invalidate-geocache")
@main.route("/invalidate-geocache.html")
def invalidate_geocache():
    geocache = 'geocache.db'

    unlink(geocache)

    return "Geocache invalidated."
15e57dca   Antoine Goutenoir   Naming things and...
238
239


4392f295   Goutte   Update the Estima...
240
@main.route("/compute")
51f564d3   Antoine Goutenoir   Add a big chunk o...
241
def compute():  # process the queue of estimation requests
24f55cde   Antoine Goutenoir   Geocode destinati...
242

dca2b847   Antoine Goutenoir   Cap the amount of...
243
244
    maximum_addresses_to_compute = 30000

24f55cde   Antoine Goutenoir   Geocode destinati...
245
246
247
248
    def _respond(_msg):
        return "<pre>%s</pre>" % _msg

    def _handle_failure(_estimation, _failure_message):
a4c03d8e   Antoine Goutenoir   Add the controlle...
249
        _estimation.status = StatusEnum.failure
70aa301f   Antoine Goutenoir   Implement another...
250
        _estimation.errors = _failure_message
24f55cde   Antoine Goutenoir   Geocode destinati...
251
252
        db.session.commit()

72460978   Antoine Goutenoir   Use warnings inst...
253
254
255
256
    def _handle_warning(_estimation, _warning_message):
        _estimation.warnings = _warning_message
        db.session.commit()

59125398   Antoine Goutenoir   Improve resilience.
257
258
    try:
        response = ""
4392f295   Goutte   Update the Estima...
259

59125398   Antoine Goutenoir   Improve resilience.
260
261
262
        count_working = Estimation.query \
            .filter_by(status=StatusEnum.working) \
            .count()
03c194bf   Antoine Goutenoir   Actually implemen...
263

59125398   Antoine Goutenoir   Improve resilience.
264
265
        if 0 < count_working:
            return _respond("Already working on estimation.")
03c194bf   Antoine Goutenoir   Actually implemen...
266

59125398   Antoine Goutenoir   Improve resilience.
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
        try:
            estimation = Estimation.query \
                .filter_by(status=StatusEnum.pending) \
                .order_by(Estimation.id.asc()) \
                .first()
        except sqlalchemy.orm.exc.NoResultFound:
            return _respond("No estimation in the queue.")
        except Exception as e:
            return _respond("Database error: %s" % (e,))

        if not estimation:
            return _respond("No estimation in the queue.")

        estimation.status = StatusEnum.working
        db.session.commit()
51f564d3   Antoine Goutenoir   Add a big chunk o...
282

59125398   Antoine Goutenoir   Improve resilience.
283
284
285
        response += u"Processing estimation `%s`...\n" % (
            estimation.public_id
        )
24f55cde   Antoine Goutenoir   Geocode destinati...
286

59125398   Antoine Goutenoir   Improve resilience.
287
288
        failed_addresses = []
        geocoder = CachedGeocoder()
03c194bf   Antoine Goutenoir   Actually implemen...
289

59125398   Antoine Goutenoir   Improve resilience.
290
        # GEOCODE ORIGINS #########################################################
24f55cde   Antoine Goutenoir   Geocode destinati...
291

59125398   Antoine Goutenoir   Improve resilience.
292
        origins_addresses = estimation.origin_addresses.strip().split("\n")
dca2b847   Antoine Goutenoir   Cap the amount of...
293
        origins_addresses_count = len(origins_addresses)
59125398   Antoine Goutenoir   Improve resilience.
294
        origins = []
24f55cde   Antoine Goutenoir   Geocode destinati...
295

dca2b847   Antoine Goutenoir   Cap the amount of...
296
        if origins_addresses_count > maximum_addresses_to_compute:
e144bb1b   Antoine Goutenoir   Fix encoding.
297
            errmsg = u"Too many origins. (%d > %d)  Please contact us for support of that many origins." % (origins_addresses_count, maximum_addresses_to_compute)
dca2b847   Antoine Goutenoir   Cap the amount of...
298
299
300
301
            _handle_failure(estimation, errmsg)
            return _respond(errmsg)

        for i in range(origins_addresses_count):
51f564d3   Antoine Goutenoir   Add a big chunk o...
302

59125398   Antoine Goutenoir   Improve resilience.
303
304
305
            origin_address = origins_addresses[i].strip()
            if origin_address in failed_addresses:
                continue
51f564d3   Antoine Goutenoir   Add a big chunk o...
306

59125398   Antoine Goutenoir   Improve resilience.
307
308
309
310
311
312
313
314
315
            try:
                origin = geocoder.geocode(origin_address.encode('utf-8'))
            except geopy.exc.GeopyError as e:
                response += u"Failed to geocode origin `%s`.\n%s\n" % (
                    origin_address, e,
                )
                _handle_warning(estimation, response)
                failed_addresses.append(origin_address)
                continue
51f564d3   Antoine Goutenoir   Add a big chunk o...
316

59125398   Antoine Goutenoir   Improve resilience.
317
318
319
320
321
322
323
            if origin is None:
                response += u"Failed to geocode origin `%s`.\n" % (
                    origin_address,
                )
                _handle_warning(estimation, response)
                failed_addresses.append(origin_address)
                continue
51f564d3   Antoine Goutenoir   Add a big chunk o...
324

59125398   Antoine Goutenoir   Improve resilience.
325
            origins.append(origin)
51f564d3   Antoine Goutenoir   Add a big chunk o...
326

59125398   Antoine Goutenoir   Improve resilience.
327
328
329
            response += u"Origin: %s == %s (%f, %f)\n" % (
                origin_address, origin.address,
                origin.latitude, origin.longitude,
51f564d3   Antoine Goutenoir   Add a big chunk o...
330
            )
51f564d3   Antoine Goutenoir   Add a big chunk o...
331

59125398   Antoine Goutenoir   Improve resilience.
332
        # GEOCODE DESTINATIONS ####################################################
51f564d3   Antoine Goutenoir   Add a big chunk o...
333

59125398   Antoine Goutenoir   Improve resilience.
334
        destinations_addresses = estimation.destination_addresses.strip().split("\n")
dca2b847   Antoine Goutenoir   Cap the amount of...
335
        destinations_addresses_count = len(destinations_addresses)
59125398   Antoine Goutenoir   Improve resilience.
336
        destinations = []
51f564d3   Antoine Goutenoir   Add a big chunk o...
337

dca2b847   Antoine Goutenoir   Cap the amount of...
338
        if destinations_addresses_count > maximum_addresses_to_compute:
e144bb1b   Antoine Goutenoir   Fix encoding.
339
            errmsg = u"Too many destinations. (%d > %d)  Please contact us for support of that many destinations." % (destinations_addresses_count, maximum_addresses_to_compute)
dca2b847   Antoine Goutenoir   Cap the amount of...
340
341
342
343
            _handle_failure(estimation, errmsg)
            return _respond(errmsg)

        for i in range(destinations_addresses_count):
51f564d3   Antoine Goutenoir   Add a big chunk o...
344

59125398   Antoine Goutenoir   Improve resilience.
345
346
347
            destination_address = destinations_addresses[i].strip()
            if destination_address in failed_addresses:
                continue
24f55cde   Antoine Goutenoir   Geocode destinati...
348

59125398   Antoine Goutenoir   Improve resilience.
349
350
351
352
353
354
355
356
357
            try:
                destination = geocoder.geocode(destination_address.encode('utf-8'))
            except geopy.exc.GeopyError as e:
                response += u"Failed to geocode destination `%s`.\n%s\n" % (
                    destination_address, e,
                )
                _handle_warning(estimation, response)
                failed_addresses.append(destination_address)
                continue
24f55cde   Antoine Goutenoir   Geocode destinati...
358

59125398   Antoine Goutenoir   Improve resilience.
359
360
361
362
363
364
365
            if destination is None:
                response += u"Failed to geocode destination `%s`.\n" % (
                    destination_address,
                )
                _handle_warning(estimation, response)
                failed_addresses.append(destination_address)
                continue
24f55cde   Antoine Goutenoir   Geocode destinati...
366

59125398   Antoine Goutenoir   Improve resilience.
367
            # print(repr(destination.raw))
24f55cde   Antoine Goutenoir   Geocode destinati...
368

59125398   Antoine Goutenoir   Improve resilience.
369
370
371
372
373
            destinations.append(destination)

            response += u"Destination: %s == %s (%f, %f)\n" % (
                destination_address, destination.address,
                destination.latitude, destination.longitude,
24f55cde   Antoine Goutenoir   Geocode destinati...
374
            )
24f55cde   Antoine Goutenoir   Geocode destinati...
375

8ae021a2   Antoine Goutenoir   Merge shelved cha...
376
377
        geocoder.close()

59125398   Antoine Goutenoir   Improve resilience.
378
        # GTFO IF NO ORIGINS OR NO DESTINATIONS ###################################
314c65e2   Antoine Goutenoir   Implement Scenari...
379

59125398   Antoine Goutenoir   Improve resilience.
380
381
382
383
384
385
386
387
        if 0 == len(origins):
            response += u"Failed to geocode all the origin(s).\n"
            _handle_failure(estimation, response)
            return _respond(response)
        if 0 == len(destinations):
            response += u"Failed to geocode all the destination(s).\n"
            _handle_failure(estimation, response)
            return _respond(response)
24f55cde   Antoine Goutenoir   Geocode destinati...
388

59125398   Antoine Goutenoir   Improve resilience.
389
        # GRAB AND CONFIGURE THE EMISSION MODELS ##################################
24f55cde   Antoine Goutenoir   Geocode destinati...
390

59125398   Antoine Goutenoir   Improve resilience.
391
392
        emission_models = estimation.get_models()
        # print(emission_models)
51f564d3   Antoine Goutenoir   Add a big chunk o...
393

59125398   Antoine Goutenoir   Improve resilience.
394
395
396
397
        extra_config = {
            'use_train_below_distance': estimation.use_train_below_km,
            # 'use_train_below_distance': 300,
        }
70aa301f   Antoine Goutenoir   Implement another...
398

59125398   Antoine Goutenoir   Improve resilience.
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
        # PREPARE RESULT DICTIONARY THAT WILL BE STORED ###########################

        results = {}

        # UTILITY PRIVATE FUNCTION(S) #############################################

        def get_city_key(_location):
            # Will this hack hold?  Suspense...
            return _location.address.split(',')[0]

            # _city_key = _location.address
            # # if 'address100' in _location.raw['address']:
            # #     _city_key = _location.raw['address']['address100']
            # if 'city' in _location.raw['address']:
            #     _city_key = _location.raw['address']['city']
            # elif 'state' in _location.raw['address']:
            #     _city_key = _location.raw['address']['state']
            # return _city_key

        def compute_one_to_many(
                _origin,
                _destinations,
                _extra_config=None
        ):
            _results = {}
            footprints = {}

            destinations_by_city_key = {}

            cities_sum_foot = {}
            cities_sum_dist = {}
            cities_dict_first_model = None
            for model in emission_models:
                cities_dict = {}
                for _destination in _destinations:
                    footprint = model.compute_travel_footprint(
                        origin_latitude=_origin.latitude,
                        origin_longitude=_origin.longitude,
                        destination_latitude=_destination.latitude,
                        destination_longitude=_destination.longitude,
                        extra_config=_extra_config,
                    )
51f564d3   Antoine Goutenoir   Add a big chunk o...
441

59125398   Antoine Goutenoir   Improve resilience.
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
                    _key = get_city_key(_destination)

                    destinations_by_city_key[_key] = _destination

                    if _key not in cities_dict:
                        cities_dict[_key] = {
                            'city': _key,
                            'address': _destination.address,
                            'footprint': 0.0,
                            'distance': 0.0,
                            'train_trips': 0,
                            'plane_trips': 0,
                        }
                    cities_dict[_key]['footprint'] += footprint['co2eq_kg']
                    cities_dict[_key]['distance'] += footprint['distance_km']
                    cities_dict[_key]['train_trips'] += footprint['train_trips']
                    cities_dict[_key]['plane_trips'] += footprint['plane_trips']
                    if _key not in cities_sum_foot:
                        cities_sum_foot[_key] = 0.0
                    cities_sum_foot[_key] += footprint['co2eq_kg']
                    if _key not in cities_sum_dist:
                        cities_sum_dist[_key] = 0.0
                    cities_sum_dist[_key] += footprint['distance_km']

                cities = sorted(cities_dict.values(), key=lambda c: c['footprint'])

                footprints[model.slug] = {
                    'cities': cities,
                }

                if cities_dict_first_model is None:
                    cities_dict_first_model = deepcopy(cities_dict)

            _results['footprints'] = footprints

            total_foot = 0.0
            total_dist = 0.0
            total_train_trips = 0
            total_plane_trips = 0

            cities_mean_dict = {}
            for city in cities_sum_foot.keys():
                city_mean_foot = 1.0 * cities_sum_foot[city] / len(emission_models)
                city_mean_dist = 1.0 * cities_sum_dist[city] / len(emission_models)
                city_train_trips = cities_dict_first_model[city]['train_trips']
                city_plane_trips = cities_dict_first_model[city]['plane_trips']
                cities_mean_dict[city] = {
                    'address': destinations_by_city_key[city].address,
                    'city': city,
                    'footprint': city_mean_foot,
                    'distance': city_mean_dist,
                    'train_trips': city_train_trips,
                    'plane_trips': city_plane_trips,
                }
                total_foot += city_mean_foot
                total_dist += city_mean_dist
                total_train_trips += city_train_trips
                total_plane_trips += city_plane_trips

            cities_mean = [cities_mean_dict[k] for k in cities_mean_dict.keys()]
            cities_mean = sorted(cities_mean, key=lambda c: c['footprint'])

            _results['mean_footprint'] = {  # DEPRECATED?
                'cities': cities_mean
584b13cc   Antoine Goutenoir   Order the results...
506
            }
59125398   Antoine Goutenoir   Improve resilience.
507
            _results['cities'] = cities_mean
1d48272e   Antoine Goutenoir   Compute the mean ...
508

59125398   Antoine Goutenoir   Improve resilience.
509
510
            _results['total'] = total_foot  # DEPRECATED
            _results['footprint'] = total_foot
584b13cc   Antoine Goutenoir   Order the results...
511

59125398   Antoine Goutenoir   Improve resilience.
512
513
514
            _results['distance'] = total_dist
            _results['train_trips'] = total_train_trips
            _results['plane_trips'] = total_plane_trips
70aa301f   Antoine Goutenoir   Implement another...
515

59125398   Antoine Goutenoir   Improve resilience.
516
            return _results
24f55cde   Antoine Goutenoir   Geocode destinati...
517

59125398   Antoine Goutenoir   Improve resilience.
518
519
520
521
522
523
524
525
526
527
528
529
        # SCENARIO A : One Origin, At Least One Destination #######################
        #
        # In this scenario, we compute the sum of each of the travels' footprint,
        # for each of the Emission Models, and present a mean of all Models.
        #
        if 1 == len(origins):
            estimation.scenario = ScenarioEnum.one_to_many
            results = compute_one_to_many(
                _origin=origins[0],
                _destinations=destinations,
                _extra_config=extra_config,
            )
94ae2730   Antoine Goutenoir   Ignore duplicates...
530

59125398   Antoine Goutenoir   Improve resilience.
531
532
533
534
535
536
537
538
        # SCENARIO B : At Least One Origin, One Destination #######################
        #
        # Same as A for now.
        #
        elif 1 == len(destinations):
            estimation.scenario = ScenarioEnum.many_to_one
            results = compute_one_to_many(
                _origin=destinations[0],
314c65e2   Antoine Goutenoir   Implement Scenari...
539
                _destinations=origins,
8a693e06   Antoine Goutenoir   Glue the extra co...
540
                _extra_config=extra_config,
314c65e2   Antoine Goutenoir   Implement Scenari...
541
            )
314c65e2   Antoine Goutenoir   Implement Scenari...
542

59125398   Antoine Goutenoir   Improve resilience.
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
        # SCENARIO C : At Least One Origin, At Least One Destination ##############
        #
        # Run Scenario A for each Destination, and expose optimum Destination.
        #
        else:
            estimation.scenario = ScenarioEnum.many_to_many
            unique_city_keys = []
            result_cities = []
            for destination in destinations:
                city_key = get_city_key(destination)

                if city_key in unique_city_keys:
                    continue
                else:
                    unique_city_keys.append(city_key)
1d48272e   Antoine Goutenoir   Compute the mean ...
558

59125398   Antoine Goutenoir   Improve resilience.
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
                city_results = compute_one_to_many(
                    _origin=destination,
                    _destinations=origins,
                    _extra_config=extra_config,
                )
                city_results['city'] = city_key
                city_results['address'] = destination.address
                result_cities.append(city_results)

            result_cities = sorted(result_cities, key=lambda c: int(c['footprint']))
            results = {
                'cities': result_cities,
            }

        # WRITE RESULTS INTO THE DATABASE #########################################
a4c03d8e   Antoine Goutenoir   Add the controlle...
574

59125398   Antoine Goutenoir   Improve resilience.
575
576
577
578
579
        estimation.status = StatusEnum.success
        estimation.output_yaml = yaml_dump(results)
        db.session.commit()

        # FINALLY, RESPOND ########################################################
a4c03d8e   Antoine Goutenoir   Add the controlle...
580

59125398   Antoine Goutenoir   Improve resilience.
581
        response += yaml_dump(results) + "\n"
a4c03d8e   Antoine Goutenoir   Add the controlle...
582

59125398   Antoine Goutenoir   Improve resilience.
583
        return _respond(response)
4392f295   Goutte   Update the Estima...
584

59125398   Antoine Goutenoir   Improve resilience.
585
    except Exception as e:
e144bb1b   Antoine Goutenoir   Fix encoding.
586
        errmsg = u"Computation failed : %s" % (e,)
ce850e3a   Antoine Goutenoir   Revert traceback.
587
        # errmsg = u"%s\n\n%s" % (errmsg, traceback.format_exc())
59125398   Antoine Goutenoir   Improve resilience.
588
589
590
        if estimation:
            _handle_failure(estimation, errmsg)
        return _respond(errmsg)
a4c03d8e   Antoine Goutenoir   Add the controlle...
591
592


b9fc86c3   Antoine Goutenoir   Secure the admin ...
593
594
@main.route("/estimation/<public_id>.<extension>")
def consult_estimation(public_id, extension):
a4c03d8e   Antoine Goutenoir   Add the controlle...
595
596
597
598
599
600
601
    try:
        estimation = Estimation.query \
            .filter_by(public_id=public_id) \
            .one()
    except sqlalchemy.orm.exc.NoResultFound:
        return abort(404)
    except Exception as e:
59125398   Antoine Goutenoir   Improve resilience.
602
        # TODO: log?
a4c03d8e   Antoine Goutenoir   Add the controlle...
603
604
605
606
607
608
        return abort(500)

    # allowed_formats = ['html']
    # if format not in allowed_formats:
    #     abort(404)

e721cb31   Antoine Goutenoir   Provide a YAML fi...
609
    unavailable_statuses = [StatusEnum.pending, StatusEnum.working]
314c65e2   Antoine Goutenoir   Implement Scenari...
610

b9fc86c3   Antoine Goutenoir   Secure the admin ...
611
    if extension in ['xhtml', 'html', 'htm']:
e721cb31   Antoine Goutenoir   Provide a YAML fi...
612
613

        if estimation.status in unavailable_statuses:
a4c03d8e   Antoine Goutenoir   Add the controlle...
614
615
616
617
618
            return render_template(
                "estimation-queue-wait.html",
                estimation=estimation
            )
        else:
40382971   Antoine Goutenoir   Add the sum of es...
619
620
            estimation_output = estimation.get_output_dict()
            estimation_sum = 0
37e28f2c   Antoine Goutenoir   Improve resilience.
621
622
623
            if estimation_output:
                for city in estimation_output['cities']:
                    estimation_sum += city['footprint']
40382971   Antoine Goutenoir   Add the sum of es...
624

a4c03d8e   Antoine Goutenoir   Add the controlle...
625
626
            return render_template(
                "estimation.html",
91751451   Antoine Goutenoir   Shift the API to ...
627
                estimation=estimation,
40382971   Antoine Goutenoir   Add the sum of es...
628
629
                estimation_output=estimation_output,
                estimation_sum=estimation_sum,
a4c03d8e   Antoine Goutenoir   Add the controlle...
630
631
            )

b9fc86c3   Antoine Goutenoir   Secure the admin ...
632
    elif extension in ['yaml', 'yml']:
e721cb31   Antoine Goutenoir   Provide a YAML fi...
633
634
635
636
637
638

        if estimation.status in unavailable_statuses:
            abort(404)

        return estimation.output_yaml

b9fc86c3   Antoine Goutenoir   Secure the admin ...
639
    elif 'csv' == extension:
a4c03d8e   Antoine Goutenoir   Add the controlle...
640

e721cb31   Antoine Goutenoir   Provide a YAML fi...
641
642
643
        if estimation.status in unavailable_statuses:
            abort(404)

a4c03d8e   Antoine Goutenoir   Add the controlle...
644
645
        si = StringIO()
        cw = csv.writer(si, quoting=csv.QUOTE_ALL)
b935618e   Antoine Goutenoir   Count the number ...
646
647
648
649
650
        cw.writerow([
            u"city", u"address",
            u"co2 (kg)", u"distance (km)",
            u"plane trips", u'train trips',
        ])
a4c03d8e   Antoine Goutenoir   Add the controlle...
651
652

        results = estimation.get_output_dict()
5634c975   Antoine Goutenoir   Expose travel dis...
653
654
655
656
657
658
        for city in results['cities']:
            cw.writerow([
                city['city'].encode(OUT_ENCODING),
                city['address'].encode(OUT_ENCODING),
                round(city['footprint'], 3),
                round(city['distance'], 3),
b935618e   Antoine Goutenoir   Count the number ...
659
660
                city['plane_trips'],
                city['train_trips'],
5634c975   Antoine Goutenoir   Expose travel dis...
661
662
            ])

67f85bce   Antoine Goutenoir   Generate a CSV fi...
663
664
665
666
667
668
669
670
        # return si.getvalue().strip('\r\n')
        return Response(
            response=si.getvalue().strip('\r\n'),
            headers={
                'Content-type': 'text/csv',
                'Content-disposition': "attachment; filename=%s.csv"%public_id,
            },
        )
a4c03d8e   Antoine Goutenoir   Add the controlle...
671
672
673

    else:
        abort(404)
b9fc86c3   Antoine Goutenoir   Secure the admin ...
674
675


67f85bce   Antoine Goutenoir   Generate a CSV fi...
676
677
@main.route("/scaling_laws.csv")
def get_scaling_laws_csv():
a728e600   Antoine Goutenoir   Allow configurati...
678
    distances = content.laws_plot.distances
67f85bce   Antoine Goutenoir   Generate a CSV fi...
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
    models = get_emission_models()

    si = StringIO()
    cw = csv.writer(si, quoting=csv.QUOTE_ALL)

    header = ['distance'] + [model.slug for model in models]
    cw.writerow(header)

    for distance in distances:
        row = [distance]
        for model in models:
            row.append(model.compute_airplane_distance_footprint(distance))
        cw.writerow(row)

    return Response(
        response=si.getvalue().strip('\r\n'),
        headers={
            'Content-type': 'text/csv',
            'Content-disposition': 'attachment; filename=scaling_laws.csv',
        },
    )


b9fc86c3   Antoine Goutenoir   Secure the admin ...
702
703
704
705
706
707
@main.route("/test")
@basic_auth.required
def dev_test():
    import os

    return os.getenv('ADMIN_USERNAME')