diff --git a/flaskr/controllers/main_controller.py b/flaskr/controllers/main_controller.py index c016504..2968a5e 100644 --- a/flaskr/controllers/main_controller.py +++ b/flaskr/controllers/main_controller.py @@ -240,318 +240,325 @@ def compute(): # process the queue of estimation requests _estimation.warnings = _warning_message db.session.commit() - response = "" + try: + response = "" - count_working = Estimation.query \ - .filter_by(status=StatusEnum.working) \ - .count() + count_working = Estimation.query \ + .filter_by(status=StatusEnum.working) \ + .count() - if 0 < count_working: - return _respond("Already working on estimation.") + if 0 < count_working: + return _respond("Already working on estimation.") - try: - estimation = Estimation.query \ - .filter_by(status=StatusEnum.pending) \ - .order_by(Estimation.id.asc()) \ - .first() - except sqlalchemy.orm.exc.NoResultFound: - return _respond("No estimation in the queue.") - except Exception as e: - return _respond("Database error: %s" % (e,)) + try: + estimation = Estimation.query \ + .filter_by(status=StatusEnum.pending) \ + .order_by(Estimation.id.asc()) \ + .first() + except sqlalchemy.orm.exc.NoResultFound: + return _respond("No estimation in the queue.") + except Exception as e: + return _respond("Database error: %s" % (e,)) + + if not estimation: + return _respond("No estimation in the queue.") + + estimation.status = StatusEnum.working + db.session.commit() - if not estimation: - return _respond("No estimation in the queue.") + response += u"Processing estimation `%s`...\n" % ( + estimation.public_id + ) - estimation.status = StatusEnum.working - db.session.commit() + failed_addresses = [] + geocoder = CachedGeocoder() - response += u"Processing estimation `%s`...\n" % ( - estimation.public_id - ) + # GEOCODE ORIGINS ######################################################### - failed_addresses = [] - geocoder = CachedGeocoder() + origins_addresses = estimation.origin_addresses.strip().split("\n") + origins = [] - # GEOCODE ORIGINS ######################################################### + for i in range(len(origins_addresses)): - origins_addresses = estimation.origin_addresses.strip().split("\n") - origins = [] + origin_address = origins_addresses[i].strip() + if origin_address in failed_addresses: + continue - for i in range(len(origins_addresses)): + try: + origin = geocoder.geocode(origin_address.encode('utf-8')) + except geopy.exc.GeopyError as e: + response += u"Failed to geocode origin `%s`.\n%s\n" % ( + origin_address, e, + ) + _handle_warning(estimation, response) + failed_addresses.append(origin_address) + continue - origin_address = origins_addresses[i].strip() - if origin_address in failed_addresses: - continue + if origin is None: + response += u"Failed to geocode origin `%s`.\n" % ( + origin_address, + ) + _handle_warning(estimation, response) + failed_addresses.append(origin_address) + continue - try: - origin = geocoder.geocode(origin_address.encode('utf-8')) - except geopy.exc.GeopyError as e: - response += u"Failed to geocode origin `%s`.\n%s\n" % ( - origin_address, e, - ) - _handle_warning(estimation, response) - failed_addresses.append(origin_address) - continue + origins.append(origin) - if origin is None: - response += u"Failed to geocode origin `%s`.\n" % ( - origin_address, + response += u"Origin: %s == %s (%f, %f)\n" % ( + origin_address, origin.address, + origin.latitude, origin.longitude, ) - _handle_warning(estimation, response) - failed_addresses.append(origin_address) - continue - origins.append(origin) + # GEOCODE DESTINATIONS #################################################### - response += u"Origin: %s == %s (%f, %f)\n" % ( - origin_address, origin.address, - origin.latitude, origin.longitude, - ) + destinations_addresses = estimation.destination_addresses.strip().split("\n") + destinations = [] - # GEOCODE DESTINATIONS #################################################### + for i in range(len(destinations_addresses)): - destinations_addresses = estimation.destination_addresses.strip().split("\n") - destinations = [] + destination_address = destinations_addresses[i].strip() + if destination_address in failed_addresses: + continue - for i in range(len(destinations_addresses)): + try: + destination = geocoder.geocode(destination_address.encode('utf-8')) + except geopy.exc.GeopyError as e: + response += u"Failed to geocode destination `%s`.\n%s\n" % ( + destination_address, e, + ) + _handle_warning(estimation, response) + failed_addresses.append(destination_address) + continue - destination_address = destinations_addresses[i].strip() - if destination_address in failed_addresses: - continue + if destination is None: + response += u"Failed to geocode destination `%s`.\n" % ( + destination_address, + ) + _handle_warning(estimation, response) + failed_addresses.append(destination_address) + continue - try: - destination = geocoder.geocode(destination_address.encode('utf-8')) - except geopy.exc.GeopyError as e: - response += u"Failed to geocode destination `%s`.\n%s\n" % ( - destination_address, e, - ) - _handle_warning(estimation, response) - failed_addresses.append(destination_address) - continue + # print(repr(destination.raw)) - if destination is None: - response += u"Failed to geocode destination `%s`.\n" % ( - destination_address, + destinations.append(destination) + + response += u"Destination: %s == %s (%f, %f)\n" % ( + destination_address, destination.address, + destination.latitude, destination.longitude, ) - _handle_warning(estimation, response) - failed_addresses.append(destination_address) - continue - # print(repr(destination.raw)) + # GTFO IF NO ORIGINS OR NO DESTINATIONS ################################### - destinations.append(destination) + if 0 == len(origins): + response += u"Failed to geocode all the origin(s).\n" + _handle_failure(estimation, response) + return _respond(response) + if 0 == len(destinations): + response += u"Failed to geocode all the destination(s).\n" + _handle_failure(estimation, response) + return _respond(response) - response += u"Destination: %s == %s (%f, %f)\n" % ( - destination_address, destination.address, - destination.latitude, destination.longitude, - ) + # GRAB AND CONFIGURE THE EMISSION MODELS ################################## - # GTFO IF NO ORIGINS OR NO DESTINATIONS ################################### + emission_models = estimation.get_models() + # print(emission_models) - if 0 == len(origins): - response += u"Failed to geocode all the origin(s).\n" - _handle_failure(estimation, response) - return _respond(response) - if 0 == len(destinations): - response += u"Failed to geocode all the destination(s).\n" - _handle_failure(estimation, response) - return _respond(response) - - # GRAB AND CONFIGURE THE EMISSION MODELS ################################## - - emission_models = estimation.get_models() - # print(emission_models) - - extra_config = { - 'use_train_below_distance': estimation.use_train_below_km, - # 'use_train_below_distance': 300, - } - - # PREPARE RESULT DICTIONARY THAT WILL BE STORED ########################### - - results = {} - - # UTILITY PRIVATE FUNCTION(S) ############################################# - - def get_city_key(_location): - # Will this hack hold? Suspense... - return _location.address.split(',')[0] - - # _city_key = _location.address - # # if 'address100' in _location.raw['address']: - # # _city_key = _location.raw['address']['address100'] - # if 'city' in _location.raw['address']: - # _city_key = _location.raw['address']['city'] - # elif 'state' in _location.raw['address']: - # _city_key = _location.raw['address']['state'] - # return _city_key - - def compute_one_to_many( - _origin, - _destinations, - _extra_config=None - ): - _results = {} - footprints = {} - - destinations_by_city_key = {} - - cities_sum_foot = {} - cities_sum_dist = {} - cities_dict_first_model = None - for model in emission_models: - cities_dict = {} - for _destination in _destinations: - footprint = model.compute_travel_footprint( - origin_latitude=_origin.latitude, - origin_longitude=_origin.longitude, - destination_latitude=_destination.latitude, - destination_longitude=_destination.longitude, - extra_config=_extra_config, - ) + extra_config = { + 'use_train_below_distance': estimation.use_train_below_km, + # 'use_train_below_distance': 300, + } - _key = get_city_key(_destination) - - destinations_by_city_key[_key] = _destination - - if _key not in cities_dict: - cities_dict[_key] = { - 'city': _key, - 'address': _destination.address, - 'footprint': 0.0, - 'distance': 0.0, - 'train_trips': 0, - 'plane_trips': 0, - } - cities_dict[_key]['footprint'] += footprint['co2eq_kg'] - cities_dict[_key]['distance'] += footprint['distance_km'] - cities_dict[_key]['train_trips'] += footprint['train_trips'] - cities_dict[_key]['plane_trips'] += footprint['plane_trips'] - if _key not in cities_sum_foot: - cities_sum_foot[_key] = 0.0 - cities_sum_foot[_key] += footprint['co2eq_kg'] - if _key not in cities_sum_dist: - cities_sum_dist[_key] = 0.0 - cities_sum_dist[_key] += footprint['distance_km'] - - cities = sorted(cities_dict.values(), key=lambda c: c['footprint']) - - footprints[model.slug] = { - 'cities': cities, - } + # PREPARE RESULT DICTIONARY THAT WILL BE STORED ########################### + + results = {} + + # UTILITY PRIVATE FUNCTION(S) ############################################# + + def get_city_key(_location): + # Will this hack hold? Suspense... + return _location.address.split(',')[0] + + # _city_key = _location.address + # # if 'address100' in _location.raw['address']: + # # _city_key = _location.raw['address']['address100'] + # if 'city' in _location.raw['address']: + # _city_key = _location.raw['address']['city'] + # elif 'state' in _location.raw['address']: + # _city_key = _location.raw['address']['state'] + # return _city_key + + def compute_one_to_many( + _origin, + _destinations, + _extra_config=None + ): + _results = {} + footprints = {} + + destinations_by_city_key = {} + + cities_sum_foot = {} + cities_sum_dist = {} + cities_dict_first_model = None + for model in emission_models: + cities_dict = {} + for _destination in _destinations: + footprint = model.compute_travel_footprint( + origin_latitude=_origin.latitude, + origin_longitude=_origin.longitude, + destination_latitude=_destination.latitude, + destination_longitude=_destination.longitude, + extra_config=_extra_config, + ) - if cities_dict_first_model is None: - cities_dict_first_model = deepcopy(cities_dict) - - _results['footprints'] = footprints - - total_foot = 0.0 - total_dist = 0.0 - total_train_trips = 0 - total_plane_trips = 0 - - cities_mean_dict = {} - for city in cities_sum_foot.keys(): - city_mean_foot = 1.0 * cities_sum_foot[city] / len(emission_models) - city_mean_dist = 1.0 * cities_sum_dist[city] / len(emission_models) - city_train_trips = cities_dict_first_model[city]['train_trips'] - city_plane_trips = cities_dict_first_model[city]['plane_trips'] - cities_mean_dict[city] = { - 'address': destinations_by_city_key[city].address, - 'city': city, - 'footprint': city_mean_foot, - 'distance': city_mean_dist, - 'train_trips': city_train_trips, - 'plane_trips': city_plane_trips, + _key = get_city_key(_destination) + + destinations_by_city_key[_key] = _destination + + if _key not in cities_dict: + cities_dict[_key] = { + 'city': _key, + 'address': _destination.address, + 'footprint': 0.0, + 'distance': 0.0, + 'train_trips': 0, + 'plane_trips': 0, + } + cities_dict[_key]['footprint'] += footprint['co2eq_kg'] + cities_dict[_key]['distance'] += footprint['distance_km'] + cities_dict[_key]['train_trips'] += footprint['train_trips'] + cities_dict[_key]['plane_trips'] += footprint['plane_trips'] + if _key not in cities_sum_foot: + cities_sum_foot[_key] = 0.0 + cities_sum_foot[_key] += footprint['co2eq_kg'] + if _key not in cities_sum_dist: + cities_sum_dist[_key] = 0.0 + cities_sum_dist[_key] += footprint['distance_km'] + + cities = sorted(cities_dict.values(), key=lambda c: c['footprint']) + + footprints[model.slug] = { + 'cities': cities, + } + + if cities_dict_first_model is None: + cities_dict_first_model = deepcopy(cities_dict) + + _results['footprints'] = footprints + + total_foot = 0.0 + total_dist = 0.0 + total_train_trips = 0 + total_plane_trips = 0 + + cities_mean_dict = {} + for city in cities_sum_foot.keys(): + city_mean_foot = 1.0 * cities_sum_foot[city] / len(emission_models) + city_mean_dist = 1.0 * cities_sum_dist[city] / len(emission_models) + city_train_trips = cities_dict_first_model[city]['train_trips'] + city_plane_trips = cities_dict_first_model[city]['plane_trips'] + cities_mean_dict[city] = { + 'address': destinations_by_city_key[city].address, + 'city': city, + 'footprint': city_mean_foot, + 'distance': city_mean_dist, + 'train_trips': city_train_trips, + 'plane_trips': city_plane_trips, + } + total_foot += city_mean_foot + total_dist += city_mean_dist + total_train_trips += city_train_trips + total_plane_trips += city_plane_trips + + cities_mean = [cities_mean_dict[k] for k in cities_mean_dict.keys()] + cities_mean = sorted(cities_mean, key=lambda c: c['footprint']) + + _results['mean_footprint'] = { # DEPRECATED? + 'cities': cities_mean } - total_foot += city_mean_foot - total_dist += city_mean_dist - total_train_trips += city_train_trips - total_plane_trips += city_plane_trips + _results['cities'] = cities_mean - cities_mean = [cities_mean_dict[k] for k in cities_mean_dict.keys()] - cities_mean = sorted(cities_mean, key=lambda c: c['footprint']) + _results['total'] = total_foot # DEPRECATED + _results['footprint'] = total_foot - _results['mean_footprint'] = { # DEPRECATED? - 'cities': cities_mean - } - _results['cities'] = cities_mean - - _results['total'] = total_foot # DEPRECATED - _results['footprint'] = total_foot - - _results['distance'] = total_dist - _results['train_trips'] = total_train_trips - _results['plane_trips'] = total_plane_trips - - return _results - - # SCENARIO A : One Origin, At Least One Destination ####################### - # - # In this scenario, we compute the sum of each of the travels' footprint, - # for each of the Emission Models, and present a mean of all Models. - # - if 1 == len(origins): - estimation.scenario = ScenarioEnum.one_to_many - results = compute_one_to_many( - _origin=origins[0], - _destinations=destinations, - _extra_config=extra_config, - ) + _results['distance'] = total_dist + _results['train_trips'] = total_train_trips + _results['plane_trips'] = total_plane_trips - # SCENARIO B : At Least One Origin, One Destination ####################### - # - # Same as A for now. - # - elif 1 == len(destinations): - estimation.scenario = ScenarioEnum.many_to_one - results = compute_one_to_many( - _origin=destinations[0], - _destinations=origins, - _extra_config=extra_config, - ) + return _results - # SCENARIO C : At Least One Origin, At Least One Destination ############## - # - # Run Scenario A for each Destination, and expose optimum Destination. - # - else: - estimation.scenario = ScenarioEnum.many_to_many - unique_city_keys = [] - result_cities = [] - for destination in destinations: - city_key = get_city_key(destination) - - if city_key in unique_city_keys: - continue - else: - unique_city_keys.append(city_key) + # SCENARIO A : One Origin, At Least One Destination ####################### + # + # In this scenario, we compute the sum of each of the travels' footprint, + # for each of the Emission Models, and present a mean of all Models. + # + if 1 == len(origins): + estimation.scenario = ScenarioEnum.one_to_many + results = compute_one_to_many( + _origin=origins[0], + _destinations=destinations, + _extra_config=extra_config, + ) - city_results = compute_one_to_many( - _origin=destination, + # SCENARIO B : At Least One Origin, One Destination ####################### + # + # Same as A for now. + # + elif 1 == len(destinations): + estimation.scenario = ScenarioEnum.many_to_one + results = compute_one_to_many( + _origin=destinations[0], _destinations=origins, _extra_config=extra_config, ) - city_results['city'] = city_key - city_results['address'] = destination.address - result_cities.append(city_results) - result_cities = sorted(result_cities, key=lambda c: int(c['footprint'])) - results = { - 'cities': result_cities, - } + # SCENARIO C : At Least One Origin, At Least One Destination ############## + # + # Run Scenario A for each Destination, and expose optimum Destination. + # + else: + estimation.scenario = ScenarioEnum.many_to_many + unique_city_keys = [] + result_cities = [] + for destination in destinations: + city_key = get_city_key(destination) + + if city_key in unique_city_keys: + continue + else: + unique_city_keys.append(city_key) - # WRITE RESULTS INTO THE DATABASE ######################################### + city_results = compute_one_to_many( + _origin=destination, + _destinations=origins, + _extra_config=extra_config, + ) + city_results['city'] = city_key + city_results['address'] = destination.address + result_cities.append(city_results) + + result_cities = sorted(result_cities, key=lambda c: int(c['footprint'])) + results = { + 'cities': result_cities, + } + + # WRITE RESULTS INTO THE DATABASE ######################################### - estimation.status = StatusEnum.success - estimation.output_yaml = yaml_dump(results) - db.session.commit() + estimation.status = StatusEnum.success + estimation.output_yaml = yaml_dump(results) + db.session.commit() + + # FINALLY, RESPOND ######################################################## - # FINALLY, RESPOND ######################################################## + response += yaml_dump(results) + "\n" - response += yaml_dump(results) + "\n" + return _respond(response) - return _respond(response) + except Exception as e: + errmsg = "Computation failed : %s" % e + if estimation: + _handle_failure(estimation, errmsg) + return _respond(errmsg) @main.route("/estimation/.") @@ -563,7 +570,7 @@ def consult_estimation(public_id, extension): except sqlalchemy.orm.exc.NoResultFound: return abort(404) except Exception as e: - # TODO: log + # TODO: log? return abort(500) # allowed_formats = ['html'] -- libgit2 0.21.2