Blame view

flaskr/controllers/main_controller.py 23.4 KB
3ac083cb   Antoine Goutenoir   Add more details ...
1
import traceback
b935618e   Antoine Goutenoir   Count the number ...
2
3
from copy import deepcopy

314c65e2   Antoine Goutenoir   Implement Scenari...
4
5
import geopy
import sqlalchemy
51f564d3   Antoine Goutenoir   Add a big chunk o...
6

a4c03d8e   Antoine Goutenoir   Add the controlle...
7
from flask import Blueprint, render_template, flash, request, redirect, \
67f85bce   Antoine Goutenoir   Generate a CSV fi...
8
    url_for, abort, send_from_directory, Response
461850db   Antoine Goutenoir   Yet another joyfu...
9
from os.path import join
7b4d2926   Goutte   Rework the contro...
10

8ae021a2   Antoine Goutenoir   Merge shelved cha...
11
12
from os import unlink

b9fc86c3   Antoine Goutenoir   Secure the admin ...
13
from flaskr.extensions import cache, basic_auth
7b4d2926   Goutte   Rework the contro...
14
from flaskr.forms import LoginForm, EstimateForm
a1f12452   Antoine Goutenoir   Add the new conte...
15
from flaskr.models import db, User, Estimation, StatusEnum, ScenarioEnum
24f55cde   Antoine Goutenoir   Geocode destinati...
16
from flaskr.geocoder import CachedGeocoder
7b4d2926   Goutte   Rework the contro...
17

78eb2a62   Antoine Goutenoir   Use the counter.
18
19
from flaskr.core import generate_unique_id, \
    get_emission_models, increment_hit_counter
51f564d3   Antoine Goutenoir   Add a big chunk o...
20
from flaskr.content import content
7b4d2926   Goutte   Rework the contro...
21

f2fbbb72   Antoine Goutenoir   Display a nice er...
22
23
from wtforms import validators

70aa301f   Antoine Goutenoir   Implement another...
24
25
from yaml import safe_dump as yaml_dump

67f85bce   Antoine Goutenoir   Generate a CSV fi...
26
27
28
import csv
# from io import StringIO
from cStringIO import StringIO
24f55cde   Antoine Goutenoir   Geocode destinati...
29

9e44bb98   Antoine Goutenoir   Support file uplo...
30
31
32
import pandas
from pandas.compat import StringIO as PandasStringIO

7b4d2926   Goutte   Rework the contro...
33
34
35
main = Blueprint('main', __name__)


9621b8a5   Antoine Goutenoir   Fix the CSV gener...
36
37
38
OUT_ENCODING = 'utf-8'


67f85bce   Antoine Goutenoir   Generate a CSV fi...
39
# -----------------------------------------------------------------------------
78eb2a62   Antoine Goutenoir   Use the counter.
40
# -----------------------------------------------------------------------------
67f85bce   Antoine Goutenoir   Generate a CSV fi...
41
42


461850db   Antoine Goutenoir   Yet another joyfu...
43
@main.route('/favicon.ico')
7f7c6b10   Antoine Goutenoir   Disable RFI in th...
44
@cache.cached(timeout=10000)
461850db   Antoine Goutenoir   Yet another joyfu...
45
46
47
48
49
50
51
def favicon():  # we want it served from the root, not from static/
    return send_from_directory(
        join(main.root_path, '..', 'static', 'img'),
        'favicon.ico', mimetype='image/vnd.microsoft.icon'
    )


7b4d2926   Goutte   Rework the contro...
52
@main.route('/')
b0ffb1ba   Antoine Goutenoir   Review actively.
53
54
@main.route('/home')
@main.route('/home.html')
38375935   Antoine Goutenoir   Remove the cache ...
55
# @cache.cached(timeout=1000)
7b4d2926   Goutte   Rework the contro...
56
def home():
4c862b54   Antoine Goutenoir   Add a grouped bar...
57
    models = get_emission_models()
a3e9d0fc   Antoine Goutenoir   Fix home plot leg...
58
59
60
    models_dict = {}
    for model in models:
        models_dict[model.slug] = model.__dict__
78eb2a62   Antoine Goutenoir   Use the counter.
61
    increment_hit_counter()
4c862b54   Antoine Goutenoir   Add a grouped bar...
62
63
    return render_template(
        'home.html',
a3e9d0fc   Antoine Goutenoir   Fix home plot leg...
64
        models=models_dict,
4c862b54   Antoine Goutenoir   Add a grouped bar...
65
        colors=[model.color for model in models],
a3e9d0fc   Antoine Goutenoir   Fix home plot leg...
66
        labels=[model.name for model in models],
4c862b54   Antoine Goutenoir   Add a grouped bar...
67
    )
7b4d2926   Goutte   Rework the contro...
68
69


9e44bb98   Antoine Goutenoir   Support file uplo...
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
def gather_addresses(from_list, from_file):
    addresses = []
    if from_file:
        file_mimetype = from_file.mimetype
        file_contents = from_file.read()

        rows_dicts = None

        if 'text/csv' == file_mimetype:

            rows_dicts = pandas \
                .read_csv(PandasStringIO(file_contents)) \
                .rename(str.lower, axis='columns') \
                .to_dict(orient="row")

        # Here are just *some* of the mimetypes that Microsoft's
        # garbage spreadsheet files may have.
        # application/vnd.ms-excel (official)
        # application/msexcel
        # application/x-msexcel
        # application/x-ms-excel
        # application/x-excel
        # application/x-dos_ms_excel
        # application/xls
        # application/x-xls
        # application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
        # ... Let's check extension instead.

        elif from_file.filename.endswith('xls') \
                or from_file.filename.endswith('xlsx'):

            rows_dicts = pandas \
                .read_excel(PandasStringIO(file_contents)) \
                .rename(str.lower, axis='columns') \
                .to_dict(orient="row")

        # Python 3.7 only
        # elif from_file.filename.endswith('ods'):
        #
        #     rows_dicts = read_ods(PandasStringIO(file_contents), 1) \
        #         .rename(str.lower, axis='columns') \
        #         .to_dict(orient="row")

        if rows_dicts is not None:
            for row_dict in rows_dicts:
                if 'address' in row_dict:
                    addresses.append(row_dict['address'])
                    continue
                address = None
                if 'city' in row_dict:
                    address = row_dict['city']
                if 'country' in row_dict:
                    if address is None:
                        address = row_dict['country']
                    else:
                        address += "," + row_dict['country']
                if address is not None:
                    addresses.append(address)
                else:
f2fbbb72   Antoine Goutenoir   Display a nice er...
129
130
131
                    raise validators.ValidationError(
                        "We could not find Address data in the spreadsheet."
                    )
9e44bb98   Antoine Goutenoir   Support file uplo...
132
        else:
f2fbbb72   Antoine Goutenoir   Display a nice er...
133
134
135
            raise validators.ValidationError(
                "We could not find any data in the spreadsheet."
            )
9e44bb98   Antoine Goutenoir   Support file uplo...
136
137
138
139

    else:
        addresses = from_list.replace("\r", '').split("\n")

68da7cd4   Antoine Goutenoir   Improve robustnes...
140
141
142
143
144
145
146
147
148
149
150
151
    clean_addresses = []
    for address in addresses:
        if not address:
            continue
        elif type(address).__name__ == 'str':
            clean_addresses.append(unicode(address, 'utf-8'))
        else:
            clean_addresses.append(address)
    addresses = clean_addresses

    # Remove empty lines (if any) and white characters
    addresses = [a.strip() for a in addresses if a]
8ec0ce68   Antoine Goutenoir   Improve robustnes...
152

9e44bb98   Antoine Goutenoir   Support file uplo...
153
154
155
    return "\n".join(addresses)


7b4d2926   Goutte   Rework the contro...
156
@main.route("/estimate", methods=["GET", "POST"])
b0ffb1ba   Antoine Goutenoir   Review actively.
157
@main.route("/estimate.html", methods=["GET", "POST"])
7b4d2926   Goutte   Rework the contro...
158
def estimate():
77e86148   Antoine Goutenoir   Fake support for ...
159
    models = get_emission_models()
7b4d2926   Goutte   Rework the contro...
160
161
    form = EstimateForm()

f2fbbb72   Antoine Goutenoir   Display a nice er...
162
163
164
    def show_form():
        return render_template("estimate.html", form=form, models=models)

7b4d2926   Goutte   Rework the contro...
165
166
    if form.validate_on_submit():

7b4d2926   Goutte   Rework the contro...
167
        estimation = Estimation()
a4c03d8e   Antoine Goutenoir   Add the controlle...
168
        # estimation.email = form.email.data
8ae021a2   Antoine Goutenoir   Merge shelved cha...
169
        estimation.run_name = form.run_name.data
7b4d2926   Goutte   Rework the contro...
170
171
        estimation.first_name = form.first_name.data
        estimation.last_name = form.last_name.data
16f69d07   Antoine Goutenoir   Add an (unsecured...
172
        estimation.institution = form.institution.data
24f55cde   Antoine Goutenoir   Geocode destinati...
173
        estimation.status = StatusEnum.pending
f2fbbb72   Antoine Goutenoir   Display a nice er...
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192

        try:
            estimation.origin_addresses = gather_addresses(
                form.origin_addresses.data,
                form.origin_addresses_file.data
            )
        except validators.ValidationError as e:
            form.origin_addresses_file.errors.append(e.message)
            return show_form()

        try:
            estimation.destination_addresses = gather_addresses(
                form.destination_addresses.data,
                form.destination_addresses_file.data
            )
        except validators.ValidationError as e:
            form.destination_addresses_file.errors.append(e.message)
            return show_form()

4276f1aa   Antoine Goutenoir   Support train emi...
193
        estimation.use_train_below_km = form.use_train_below_km.data
f2fbbb72   Antoine Goutenoir   Display a nice er...
194

77e86148   Antoine Goutenoir   Fake support for ...
195
196
        models_slugs = []
        for model in models:
35fbac1f   Antoine Goutenoir   Fix a blooper.
197
            if getattr(form, 'use_model_%s' % model.slug).data:
77e86148   Antoine Goutenoir   Fake support for ...
198
                models_slugs.append(model.slug)
6b9c0dd2   Antoine Goutenoir   Force Unicode.
199
        estimation.models_slugs = u"\n".join(models_slugs)
7b4d2926   Goutte   Rework the contro...
200
201
202
203
204

        db.session.add(estimation)
        db.session.commit()

        flash("Estimation request submitted successfully.", "success")
a4c03d8e   Antoine Goutenoir   Add the controlle...
205
206
207
        return redirect(url_for(
            endpoint=".consult_estimation",
            public_id=estimation.public_id,
91751451   Antoine Goutenoir   Shift the API to ...
208
            extension='html'
a4c03d8e   Antoine Goutenoir   Add the controlle...
209
        ))
7b4d2926   Goutte   Rework the contro...
210
211
        # return render_template("estimate-debrief.html", form=form)

f2fbbb72   Antoine Goutenoir   Display a nice er...
212
    return show_form()
7b4d2926   Goutte   Rework the contro...
213

4392f295   Goutte   Update the Estima...
214

15e57dca   Antoine Goutenoir   Naming things and...
215
@main.route("/invalidate")
4276f1aa   Antoine Goutenoir   Support train emi...
216
@main.route("/invalidate.html")
15e57dca   Antoine Goutenoir   Naming things and...
217
218
219
220
221
222
223
224
225
226
def invalidate():
    stuck_estimations = Estimation.query \
        .filter_by(status=StatusEnum.working) \
        .all()

    for estimation in stuck_estimations:
        estimation.status = StatusEnum.failure
        estimation.errors = "Invalidated. Try again."
        db.session.commit()

8ae021a2   Antoine Goutenoir   Merge shelved cha...
227
228
229
230
231
232
233
234
235
236
237
    return "Estimations invalidated: %d" % len(stuck_estimations)


@main.route("/invalidate-geocache")
@main.route("/invalidate-geocache.html")
def invalidate_geocache():
    geocache = 'geocache.db'

    unlink(geocache)

    return "Geocache invalidated."
15e57dca   Antoine Goutenoir   Naming things and...
238
239


4392f295   Goutte   Update the Estima...
240
@main.route("/compute")
51f564d3   Antoine Goutenoir   Add a big chunk o...
241
def compute():  # process the queue of estimation requests
24f55cde   Antoine Goutenoir   Geocode destinati...
242

dca2b847   Antoine Goutenoir   Cap the amount of...
243
244
    maximum_addresses_to_compute = 30000

24f55cde   Antoine Goutenoir   Geocode destinati...
245
246
247
248
    def _respond(_msg):
        return "<pre>%s</pre>" % _msg

    def _handle_failure(_estimation, _failure_message):
a4c03d8e   Antoine Goutenoir   Add the controlle...
249
        _estimation.status = StatusEnum.failure
70aa301f   Antoine Goutenoir   Implement another...
250
        _estimation.errors = _failure_message
24f55cde   Antoine Goutenoir   Geocode destinati...
251
252
        db.session.commit()

72460978   Antoine Goutenoir   Use warnings inst...
253
254
255
256
    def _handle_warning(_estimation, _warning_message):
        _estimation.warnings = _warning_message
        db.session.commit()

59125398   Antoine Goutenoir   Improve resilience.
257
258
    try:
        response = ""
4392f295   Goutte   Update the Estima...
259

59125398   Antoine Goutenoir   Improve resilience.
260
261
262
        count_working = Estimation.query \
            .filter_by(status=StatusEnum.working) \
            .count()
03c194bf   Antoine Goutenoir   Actually implemen...
263

59125398   Antoine Goutenoir   Improve resilience.
264
265
        if 0 < count_working:
            return _respond("Already working on estimation.")
03c194bf   Antoine Goutenoir   Actually implemen...
266

59125398   Antoine Goutenoir   Improve resilience.
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
        try:
            estimation = Estimation.query \
                .filter_by(status=StatusEnum.pending) \
                .order_by(Estimation.id.asc()) \
                .first()
        except sqlalchemy.orm.exc.NoResultFound:
            return _respond("No estimation in the queue.")
        except Exception as e:
            return _respond("Database error: %s" % (e,))

        if not estimation:
            return _respond("No estimation in the queue.")

        estimation.status = StatusEnum.working
        db.session.commit()
51f564d3   Antoine Goutenoir   Add a big chunk o...
282

59125398   Antoine Goutenoir   Improve resilience.
283
284
285
        response += u"Processing estimation `%s`...\n" % (
            estimation.public_id
        )
24f55cde   Antoine Goutenoir   Geocode destinati...
286

6b9c0dd2   Antoine Goutenoir   Force Unicode.
287
288
        # GEOCODE ADDRESSES ###################################################

59125398   Antoine Goutenoir   Improve resilience.
289
290
        failed_addresses = []
        geocoder = CachedGeocoder()
03c194bf   Antoine Goutenoir   Actually implemen...
291

59125398   Antoine Goutenoir   Improve resilience.
292
        # GEOCODE ORIGINS #########################################################
24f55cde   Antoine Goutenoir   Geocode destinati...
293

59125398   Antoine Goutenoir   Improve resilience.
294
        origins_addresses = estimation.origin_addresses.strip().split("\n")
dca2b847   Antoine Goutenoir   Cap the amount of...
295
        origins_addresses_count = len(origins_addresses)
59125398   Antoine Goutenoir   Improve resilience.
296
        origins = []
24f55cde   Antoine Goutenoir   Geocode destinati...
297

dca2b847   Antoine Goutenoir   Cap the amount of...
298
        if origins_addresses_count > maximum_addresses_to_compute:
e144bb1b   Antoine Goutenoir   Fix encoding.
299
            errmsg = u"Too many origins. (%d > %d)  Please contact us for support of that many origins." % (origins_addresses_count, maximum_addresses_to_compute)
dca2b847   Antoine Goutenoir   Cap the amount of...
300
301
302
303
            _handle_failure(estimation, errmsg)
            return _respond(errmsg)

        for i in range(origins_addresses_count):
51f564d3   Antoine Goutenoir   Add a big chunk o...
304

59125398   Antoine Goutenoir   Improve resilience.
305
306
307
            origin_address = origins_addresses[i].strip()
            if origin_address in failed_addresses:
                continue
51f564d3   Antoine Goutenoir   Add a big chunk o...
308

59125398   Antoine Goutenoir   Improve resilience.
309
310
311
312
313
314
315
316
317
            try:
                origin = geocoder.geocode(origin_address.encode('utf-8'))
            except geopy.exc.GeopyError as e:
                response += u"Failed to geocode origin `%s`.\n%s\n" % (
                    origin_address, e,
                )
                _handle_warning(estimation, response)
                failed_addresses.append(origin_address)
                continue
51f564d3   Antoine Goutenoir   Add a big chunk o...
318

59125398   Antoine Goutenoir   Improve resilience.
319
320
321
322
323
324
325
            if origin is None:
                response += u"Failed to geocode origin `%s`.\n" % (
                    origin_address,
                )
                _handle_warning(estimation, response)
                failed_addresses.append(origin_address)
                continue
51f564d3   Antoine Goutenoir   Add a big chunk o...
326

59125398   Antoine Goutenoir   Improve resilience.
327
            origins.append(origin)
51f564d3   Antoine Goutenoir   Add a big chunk o...
328

59125398   Antoine Goutenoir   Improve resilience.
329
330
331
            response += u"Origin: %s == %s (%f, %f)\n" % (
                origin_address, origin.address,
                origin.latitude, origin.longitude,
51f564d3   Antoine Goutenoir   Add a big chunk o...
332
            )
51f564d3   Antoine Goutenoir   Add a big chunk o...
333

59125398   Antoine Goutenoir   Improve resilience.
334
        # GEOCODE DESTINATIONS ####################################################
51f564d3   Antoine Goutenoir   Add a big chunk o...
335

59125398   Antoine Goutenoir   Improve resilience.
336
        destinations_addresses = estimation.destination_addresses.strip().split("\n")
dca2b847   Antoine Goutenoir   Cap the amount of...
337
        destinations_addresses_count = len(destinations_addresses)
59125398   Antoine Goutenoir   Improve resilience.
338
        destinations = []
51f564d3   Antoine Goutenoir   Add a big chunk o...
339

dca2b847   Antoine Goutenoir   Cap the amount of...
340
        if destinations_addresses_count > maximum_addresses_to_compute:
e144bb1b   Antoine Goutenoir   Fix encoding.
341
            errmsg = u"Too many destinations. (%d > %d)  Please contact us for support of that many destinations." % (destinations_addresses_count, maximum_addresses_to_compute)
dca2b847   Antoine Goutenoir   Cap the amount of...
342
343
344
345
            _handle_failure(estimation, errmsg)
            return _respond(errmsg)

        for i in range(destinations_addresses_count):
51f564d3   Antoine Goutenoir   Add a big chunk o...
346

59125398   Antoine Goutenoir   Improve resilience.
347
348
349
            destination_address = destinations_addresses[i].strip()
            if destination_address in failed_addresses:
                continue
24f55cde   Antoine Goutenoir   Geocode destinati...
350

59125398   Antoine Goutenoir   Improve resilience.
351
352
353
354
355
356
357
358
359
            try:
                destination = geocoder.geocode(destination_address.encode('utf-8'))
            except geopy.exc.GeopyError as e:
                response += u"Failed to geocode destination `%s`.\n%s\n" % (
                    destination_address, e,
                )
                _handle_warning(estimation, response)
                failed_addresses.append(destination_address)
                continue
24f55cde   Antoine Goutenoir   Geocode destinati...
360

59125398   Antoine Goutenoir   Improve resilience.
361
362
363
364
365
366
367
            if destination is None:
                response += u"Failed to geocode destination `%s`.\n" % (
                    destination_address,
                )
                _handle_warning(estimation, response)
                failed_addresses.append(destination_address)
                continue
24f55cde   Antoine Goutenoir   Geocode destinati...
368

59125398   Antoine Goutenoir   Improve resilience.
369
            # print(repr(destination.raw))
24f55cde   Antoine Goutenoir   Geocode destinati...
370

59125398   Antoine Goutenoir   Improve resilience.
371
372
373
374
375
            destinations.append(destination)

            response += u"Destination: %s == %s (%f, %f)\n" % (
                destination_address, destination.address,
                destination.latitude, destination.longitude,
24f55cde   Antoine Goutenoir   Geocode destinati...
376
            )
24f55cde   Antoine Goutenoir   Geocode destinati...
377

8ae021a2   Antoine Goutenoir   Merge shelved cha...
378
379
        geocoder.close()

59125398   Antoine Goutenoir   Improve resilience.
380
        # GTFO IF NO ORIGINS OR NO DESTINATIONS ###################################
314c65e2   Antoine Goutenoir   Implement Scenari...
381

59125398   Antoine Goutenoir   Improve resilience.
382
383
384
385
386
387
388
389
        if 0 == len(origins):
            response += u"Failed to geocode all the origin(s).\n"
            _handle_failure(estimation, response)
            return _respond(response)
        if 0 == len(destinations):
            response += u"Failed to geocode all the destination(s).\n"
            _handle_failure(estimation, response)
            return _respond(response)
24f55cde   Antoine Goutenoir   Geocode destinati...
390

59125398   Antoine Goutenoir   Improve resilience.
391
        # GRAB AND CONFIGURE THE EMISSION MODELS ##################################
24f55cde   Antoine Goutenoir   Geocode destinati...
392

59125398   Antoine Goutenoir   Improve resilience.
393
394
        emission_models = estimation.get_models()
        # print(emission_models)
51f564d3   Antoine Goutenoir   Add a big chunk o...
395

59125398   Antoine Goutenoir   Improve resilience.
396
397
398
399
        extra_config = {
            'use_train_below_distance': estimation.use_train_below_km,
            # 'use_train_below_distance': 300,
        }
70aa301f   Antoine Goutenoir   Implement another...
400

59125398   Antoine Goutenoir   Improve resilience.
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
        # PREPARE RESULT DICTIONARY THAT WILL BE STORED ###########################

        results = {}

        # UTILITY PRIVATE FUNCTION(S) #############################################

        def get_city_key(_location):
            # Will this hack hold?  Suspense...
            return _location.address.split(',')[0]

            # _city_key = _location.address
            # # if 'address100' in _location.raw['address']:
            # #     _city_key = _location.raw['address']['address100']
            # if 'city' in _location.raw['address']:
            #     _city_key = _location.raw['address']['city']
            # elif 'state' in _location.raw['address']:
            #     _city_key = _location.raw['address']['state']
            # return _city_key

        def compute_one_to_many(
                _origin,
                _destinations,
                _extra_config=None
        ):
            _results = {}
            footprints = {}

            destinations_by_city_key = {}

            cities_sum_foot = {}
            cities_sum_dist = {}
            cities_dict_first_model = None
            for model in emission_models:
                cities_dict = {}
                for _destination in _destinations:
                    footprint = model.compute_travel_footprint(
                        origin_latitude=_origin.latitude,
                        origin_longitude=_origin.longitude,
                        destination_latitude=_destination.latitude,
                        destination_longitude=_destination.longitude,
                        extra_config=_extra_config,
                    )
51f564d3   Antoine Goutenoir   Add a big chunk o...
443

59125398   Antoine Goutenoir   Improve resilience.
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
                    _key = get_city_key(_destination)

                    destinations_by_city_key[_key] = _destination

                    if _key not in cities_dict:
                        cities_dict[_key] = {
                            'city': _key,
                            'address': _destination.address,
                            'footprint': 0.0,
                            'distance': 0.0,
                            'train_trips': 0,
                            'plane_trips': 0,
                        }
                    cities_dict[_key]['footprint'] += footprint['co2eq_kg']
                    cities_dict[_key]['distance'] += footprint['distance_km']
                    cities_dict[_key]['train_trips'] += footprint['train_trips']
                    cities_dict[_key]['plane_trips'] += footprint['plane_trips']
                    if _key not in cities_sum_foot:
                        cities_sum_foot[_key] = 0.0
                    cities_sum_foot[_key] += footprint['co2eq_kg']
                    if _key not in cities_sum_dist:
                        cities_sum_dist[_key] = 0.0
                    cities_sum_dist[_key] += footprint['distance_km']

                cities = sorted(cities_dict.values(), key=lambda c: c['footprint'])

                footprints[model.slug] = {
                    'cities': cities,
                }

                if cities_dict_first_model is None:
                    cities_dict_first_model = deepcopy(cities_dict)

            _results['footprints'] = footprints

            total_foot = 0.0
            total_dist = 0.0
            total_train_trips = 0
            total_plane_trips = 0

            cities_mean_dict = {}
            for city in cities_sum_foot.keys():
                city_mean_foot = 1.0 * cities_sum_foot[city] / len(emission_models)
                city_mean_dist = 1.0 * cities_sum_dist[city] / len(emission_models)
                city_train_trips = cities_dict_first_model[city]['train_trips']
                city_plane_trips = cities_dict_first_model[city]['plane_trips']
                cities_mean_dict[city] = {
                    'address': destinations_by_city_key[city].address,
                    'city': city,
                    'footprint': city_mean_foot,
                    'distance': city_mean_dist,
                    'train_trips': city_train_trips,
                    'plane_trips': city_plane_trips,
                }
                total_foot += city_mean_foot
                total_dist += city_mean_dist
                total_train_trips += city_train_trips
                total_plane_trips += city_plane_trips

            cities_mean = [cities_mean_dict[k] for k in cities_mean_dict.keys()]
            cities_mean = sorted(cities_mean, key=lambda c: c['footprint'])

            _results['mean_footprint'] = {  # DEPRECATED?
                'cities': cities_mean
584b13cc   Antoine Goutenoir   Order the results...
508
            }
59125398   Antoine Goutenoir   Improve resilience.
509
            _results['cities'] = cities_mean
1d48272e   Antoine Goutenoir   Compute the mean ...
510

59125398   Antoine Goutenoir   Improve resilience.
511
512
            _results['total'] = total_foot  # DEPRECATED
            _results['footprint'] = total_foot
584b13cc   Antoine Goutenoir   Order the results...
513

59125398   Antoine Goutenoir   Improve resilience.
514
515
516
            _results['distance'] = total_dist
            _results['train_trips'] = total_train_trips
            _results['plane_trips'] = total_plane_trips
70aa301f   Antoine Goutenoir   Implement another...
517

59125398   Antoine Goutenoir   Improve resilience.
518
            return _results
24f55cde   Antoine Goutenoir   Geocode destinati...
519

59125398   Antoine Goutenoir   Improve resilience.
520
521
522
523
524
525
526
527
528
529
530
531
        # SCENARIO A : One Origin, At Least One Destination #######################
        #
        # In this scenario, we compute the sum of each of the travels' footprint,
        # for each of the Emission Models, and present a mean of all Models.
        #
        if 1 == len(origins):
            estimation.scenario = ScenarioEnum.one_to_many
            results = compute_one_to_many(
                _origin=origins[0],
                _destinations=destinations,
                _extra_config=extra_config,
            )
94ae2730   Antoine Goutenoir   Ignore duplicates...
532

59125398   Antoine Goutenoir   Improve resilience.
533
534
535
536
537
538
539
540
        # SCENARIO B : At Least One Origin, One Destination #######################
        #
        # Same as A for now.
        #
        elif 1 == len(destinations):
            estimation.scenario = ScenarioEnum.many_to_one
            results = compute_one_to_many(
                _origin=destinations[0],
314c65e2   Antoine Goutenoir   Implement Scenari...
541
                _destinations=origins,
8a693e06   Antoine Goutenoir   Glue the extra co...
542
                _extra_config=extra_config,
314c65e2   Antoine Goutenoir   Implement Scenari...
543
            )
314c65e2   Antoine Goutenoir   Implement Scenari...
544

59125398   Antoine Goutenoir   Improve resilience.
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
        # SCENARIO C : At Least One Origin, At Least One Destination ##############
        #
        # Run Scenario A for each Destination, and expose optimum Destination.
        #
        else:
            estimation.scenario = ScenarioEnum.many_to_many
            unique_city_keys = []
            result_cities = []
            for destination in destinations:
                city_key = get_city_key(destination)

                if city_key in unique_city_keys:
                    continue
                else:
                    unique_city_keys.append(city_key)
1d48272e   Antoine Goutenoir   Compute the mean ...
560

59125398   Antoine Goutenoir   Improve resilience.
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
                city_results = compute_one_to_many(
                    _origin=destination,
                    _destinations=origins,
                    _extra_config=extra_config,
                )
                city_results['city'] = city_key
                city_results['address'] = destination.address
                result_cities.append(city_results)

            result_cities = sorted(result_cities, key=lambda c: int(c['footprint']))
            results = {
                'cities': result_cities,
            }

        # WRITE RESULTS INTO THE DATABASE #########################################
a4c03d8e   Antoine Goutenoir   Add the controlle...
576

59125398   Antoine Goutenoir   Improve resilience.
577
        estimation.status = StatusEnum.success
6b9c0dd2   Antoine Goutenoir   Force Unicode.
578
        estimation.output_yaml = u"%s" % yaml_dump(results)
59125398   Antoine Goutenoir   Improve resilience.
579
580
581
        db.session.commit()

        # FINALLY, RESPOND ########################################################
a4c03d8e   Antoine Goutenoir   Add the controlle...
582

59125398   Antoine Goutenoir   Improve resilience.
583
        response += yaml_dump(results) + "\n"
a4c03d8e   Antoine Goutenoir   Add the controlle...
584

59125398   Antoine Goutenoir   Improve resilience.
585
        return _respond(response)
4392f295   Goutte   Update the Estima...
586

59125398   Antoine Goutenoir   Improve resilience.
587
    except Exception as e:
e144bb1b   Antoine Goutenoir   Fix encoding.
588
        errmsg = u"Computation failed : %s" % (e,)
ce850e3a   Antoine Goutenoir   Revert traceback.
589
        # errmsg = u"%s\n\n%s" % (errmsg, traceback.format_exc())
59125398   Antoine Goutenoir   Improve resilience.
590
591
592
        if estimation:
            _handle_failure(estimation, errmsg)
        return _respond(errmsg)
a4c03d8e   Antoine Goutenoir   Add the controlle...
593
594


b9fc86c3   Antoine Goutenoir   Secure the admin ...
595
596
@main.route("/estimation/<public_id>.<extension>")
def consult_estimation(public_id, extension):
a4c03d8e   Antoine Goutenoir   Add the controlle...
597
598
599
600
601
602
603
    try:
        estimation = Estimation.query \
            .filter_by(public_id=public_id) \
            .one()
    except sqlalchemy.orm.exc.NoResultFound:
        return abort(404)
    except Exception as e:
59125398   Antoine Goutenoir   Improve resilience.
604
        # TODO: log?
a4c03d8e   Antoine Goutenoir   Add the controlle...
605
606
607
608
609
610
        return abort(500)

    # allowed_formats = ['html']
    # if format not in allowed_formats:
    #     abort(404)

e721cb31   Antoine Goutenoir   Provide a YAML fi...
611
    unavailable_statuses = [StatusEnum.pending, StatusEnum.working]
314c65e2   Antoine Goutenoir   Implement Scenari...
612

b9fc86c3   Antoine Goutenoir   Secure the admin ...
613
    if extension in ['xhtml', 'html', 'htm']:
e721cb31   Antoine Goutenoir   Provide a YAML fi...
614
615

        if estimation.status in unavailable_statuses:
a4c03d8e   Antoine Goutenoir   Add the controlle...
616
617
618
619
620
            return render_template(
                "estimation-queue-wait.html",
                estimation=estimation
            )
        else:
40382971   Antoine Goutenoir   Add the sum of es...
621
622
            estimation_output = estimation.get_output_dict()
            estimation_sum = 0
37e28f2c   Antoine Goutenoir   Improve resilience.
623
624
625
            if estimation_output:
                for city in estimation_output['cities']:
                    estimation_sum += city['footprint']
40382971   Antoine Goutenoir   Add the sum of es...
626

a4c03d8e   Antoine Goutenoir   Add the controlle...
627
628
            return render_template(
                "estimation.html",
91751451   Antoine Goutenoir   Shift the API to ...
629
                estimation=estimation,
40382971   Antoine Goutenoir   Add the sum of es...
630
631
                estimation_output=estimation_output,
                estimation_sum=estimation_sum,
a4c03d8e   Antoine Goutenoir   Add the controlle...
632
633
            )

b9fc86c3   Antoine Goutenoir   Secure the admin ...
634
    elif extension in ['yaml', 'yml']:
e721cb31   Antoine Goutenoir   Provide a YAML fi...
635
636
637
638
639
640

        if estimation.status in unavailable_statuses:
            abort(404)

        return estimation.output_yaml

b9fc86c3   Antoine Goutenoir   Secure the admin ...
641
    elif 'csv' == extension:
a4c03d8e   Antoine Goutenoir   Add the controlle...
642

e721cb31   Antoine Goutenoir   Provide a YAML fi...
643
644
645
        if estimation.status in unavailable_statuses:
            abort(404)

a4c03d8e   Antoine Goutenoir   Add the controlle...
646
647
        si = StringIO()
        cw = csv.writer(si, quoting=csv.QUOTE_ALL)
b935618e   Antoine Goutenoir   Count the number ...
648
649
650
651
652
        cw.writerow([
            u"city", u"address",
            u"co2 (kg)", u"distance (km)",
            u"plane trips", u'train trips',
        ])
a4c03d8e   Antoine Goutenoir   Add the controlle...
653
654

        results = estimation.get_output_dict()
5634c975   Antoine Goutenoir   Expose travel dis...
655
656
657
658
659
660
        for city in results['cities']:
            cw.writerow([
                city['city'].encode(OUT_ENCODING),
                city['address'].encode(OUT_ENCODING),
                round(city['footprint'], 3),
                round(city['distance'], 3),
b935618e   Antoine Goutenoir   Count the number ...
661
662
                city['plane_trips'],
                city['train_trips'],
5634c975   Antoine Goutenoir   Expose travel dis...
663
664
            ])

67f85bce   Antoine Goutenoir   Generate a CSV fi...
665
666
667
668
669
670
671
672
        # return si.getvalue().strip('\r\n')
        return Response(
            response=si.getvalue().strip('\r\n'),
            headers={
                'Content-type': 'text/csv',
                'Content-disposition': "attachment; filename=%s.csv"%public_id,
            },
        )
a4c03d8e   Antoine Goutenoir   Add the controlle...
673
674
675

    else:
        abort(404)
b9fc86c3   Antoine Goutenoir   Secure the admin ...
676
677


67f85bce   Antoine Goutenoir   Generate a CSV fi...
678
679
@main.route("/scaling_laws.csv")
def get_scaling_laws_csv():
a728e600   Antoine Goutenoir   Allow configurati...
680
    distances = content.laws_plot.distances
67f85bce   Antoine Goutenoir   Generate a CSV fi...
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
    models = get_emission_models()

    si = StringIO()
    cw = csv.writer(si, quoting=csv.QUOTE_ALL)

    header = ['distance'] + [model.slug for model in models]
    cw.writerow(header)

    for distance in distances:
        row = [distance]
        for model in models:
            row.append(model.compute_airplane_distance_footprint(distance))
        cw.writerow(row)

    return Response(
        response=si.getvalue().strip('\r\n'),
        headers={
            'Content-type': 'text/csv',
            'Content-disposition': 'attachment; filename=scaling_laws.csv',
        },
    )


b9fc86c3   Antoine Goutenoir   Secure the admin ...
704
705
706
707
708
709
@main.route("/test")
@basic_auth.required
def dev_test():
    import os

    return os.getenv('ADMIN_USERNAME')