Add three retries when the request to Myriam's API fails for a reason or another…

… on the production server.

Add three retries when the request to Myriam's API fails for a reason or another…
… on the production server.
Goutte
1 parent a245c676
Showing 1 changed file with 79 additions and 5 deletions Show diff stats
web/run.py
@@ -46,13 +46,21 @@ with open(get_path(&#39;../config.yml&#39;), &#39;r&#39;) as config_file:
  
 log = logging.getLogger("HelioPropa")
 log.setLevel(logging.INFO)
-log.addHandler(logging.FileHandler(get_path('run.log')))
+logHandler = logging.FileHandler(get_path('run.log'))
+logHandler.setFormatter(logging.Formatter(
+    "%(asctime)s - %(levelname)s - %(message)s"
+))
+log.addHandler(logHandler)
  
  
 # SETUP FLASK ENGINE ##########################################################
  
 app = Flask(__name__, root_path=THIS_DIRECTORY)
 app.debug = environ.get('DEBUG') == 'true'
+if app.debug:
+    log.info("Starting Flask app in debug mode...")
+else:
+    log.info("Starting Flask app...")
  
  
 # SETUP JINJA2 TEMPLATE ENGINE ################################################
@@ -179,10 +187,26 @@ def retrieve_data(orbiter, what, started_at, stopped_at):
         startTime=started_at.isoformat(),
         stopTime=stopped_at.isoformat()
     )
-    response = urllib.urlopen(url)
-    remote_gzip_files = json.loads(response.read())
-    if not remote_gzip_files or remote_gzip_files == 'NODATASET':
+    retries = 0
+    success = False
+    remote_gzip_files = []
+    while not success and retries < 3:
+        try:
+            response = urllib.urlopen(url)
+            remote_gzip_files = json.loads(response.read())
+            if not remote_gzip_files:
+                raise Exception("Failed to fetch data at '%s'." % url)
+            if remote_gzip_files == 'NODATASET':
+                raise Exception("No dataset at '%s'." % url)
+            success = True
+        except Exception as e:
+            log.warn("Failed (%d/3) '%s' : %s" % (retries+1, url, e.message))
+        finally:
+            retries += 1
+    if not remote_gzip_files:
         abort(400, "Failed to fetch data at '%s'." % url)
+    if remote_gzip_files == 'NODATASET':
+        abort(400, "No dataset at '%s'." % url)
  
     # retriever = urllib.URLopener()  # would we need to do this every time ?
     local_gzip_files = []
@@ -211,7 +235,7 @@ def retrieve_data(orbiter, what, started_at, stopped_at):
  
  
 def generate_csv_contents(source_config, started_at, stopped_at):
-    # todo: iterate on models when there are many
+    # @todo iterate on models when there are many
     try:
         model_slug = source_config['models'][0]['slug']
     except:
@@ -356,6 +380,56 @@ def get_target_csv(source, started_at, stopped_at):
     return send_from_directory(get_path("../cache/"), filename)
  
  
+@app.route("/<targets>_<started_at>_<stopped_at>.zip")
+def download_targets_zip(targets, started_at, stopped_at):
+    """
+    Grab data and orbit data for the specified `target`,
+    rearrange it and return it as a CSV file.
+    `started_at` and `stopped_at` should be UTC.
+
+    targets: string list of targets' slugs, separated by `:`.
+
+
+    fixme
+
+
+    """
+
+    targets_confs = []
+    for target in targets.split(':').sort():
+        if not target:
+            abort(400, "Invalid targets format : `%s`." % targets)
+        targets_confs.append(get_source_config(target))
+    if 0 == len(targets_confs):
+        abort(400, "No valid targets specified. What are you doing?")
+
+    date_fmt = "%Y-%m-%dT%H:%M:%S"
+    try:
+        started_at = datetime.datetime.strptime(started_at, date_fmt)
+    except:
+        abort(400, "Invalid started_at parameter : '%s'." % started_at)
+    try:
+        stopped_at = datetime.datetime.strptime(stopped_at, date_fmt)
+    except:
+        abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at)
+
+
+    filename = "%s_%s_%s.csv" % (source,
+                                 started_at.strftime(date_fmt),
+                                 stopped_at.strftime(date_fmt))
+
+    local_csv_file = get_path("../cache/%s" % filename)
+    if not isfile(local_csv_file):
+        with open(local_csv_file, mode="w+") as f:
+            f.write(generate_csv_contents(source_config,
+                                          started_at=started_at,
+                                          stopped_at=stopped_at))
+
+    if not isfile(local_csv_file):
+        abort(500, "Could not cache CSV file at '%s'." % local_csv_file)
+
+    return send_from_directory(get_path("../cache/"), filename)
+
 # DEV TOOLS ###################################################################
  
 # @app.route("/inspect")