More bug hunting.

Goutte
1 parent 97d6cb96
Showing 3 changed files with 37 additions and 10 deletions Show diff stats
web/run.py
web/static/js/swapp.js
web/static/js/swapp.ls
@@ -47,7 +47,7 @@ FILE_DATE_FMT = &quot;%Y-%m-%dT%H:%M:%S&quot;
 # LOGGING #####################################################################
 log = logging.getLogger("HelioPropa")
-log.setLevel(logging.INFO)
+log.setLevel(logging.DEBUG)
 logHandler = logging.FileHandler(get_path('run.log'))
 logHandler.setFormatter(logging.Formatter(
     "%(asctime)s - %(levelname)s - %(message)s"
@@ -214,10 +214,16 @@ def retrieve_data(orbiter, what, started_at, stopped_at):
         abort(400, "Failed to fetch data at '%s'." % url)
     if remote_gzip_files == 'NODATASET':
         abort(400, "No dataset at '%s'." % url)
+    # if remote_gzip_files == ['OUTOFTIME']:
+    #     abort(400, "No data for time interval at '%s'." % url)
+
+    log.debug("Fetched remote gzip files list : %s." % str(remote_gzip_files))
     # retriever = urllib.URLopener()  # would we need to do this every time ?
     local_gzip_files = []
     for remote_gzip_file in remote_gzip_files:
+        if remote_gzip_file == 'OUTOFTIME':
+            continue
         # hotfix removeme @Myriam
         if remote_gzip_file.endswith('/.gz'):
             continue
@@ -227,16 +233,20 @@ def retrieve_data(orbiter, what, started_at, stopped_at):
         local_gzip_file = get_path("../cache/%s" % filename)
         local_gzip_files.append(local_gzip_file)
         if not isfile(local_gzip_file):
+            log.debug("Retrieving '%s'..." % local_gzip_file)
             urllib.urlretrieve(remote_gzip_file, local_gzip_file)
+            log.debug("Retrieved '%s'." % local_gzip_file)
     local_netc_files = []
     for local_gzip_file in local_gzip_files:
         local_netc_file = local_gzip_file[0:-3]
         local_netc_files.append(local_netc_file)
+        log.debug("Unzipping '%s'..." % local_gzip_file)
         with gzip.open(local_gzip_file, 'rb') as f:
             file_content = f.read()
             with open(local_netc_file, 'w+b') as g:
                 g.write(file_content)
+        log.debug("Unzipped '%s'." % local_gzip_file)
     return local_netc_files
@@ -251,9 +261,11 @@ def generate_csv_contents(source_config, started_at, stopped_at):
     # Grab the list of netCDF files from Myriam's API
     # http://cdpp.irap.omp.eu/BASE/DDService/getDataUrl.php?dataSet=jupiter_orb_all&StartTime=2014-02-23T10:00:10&StopTime=2017-02-24T23:59:00
     # http://cdpp.irap.omp.eu/BASE/DATA/TAO/JUPITER/SW/sw_2014.nc.gz
+    log.info("Generating CSV for '%s'..." % source_config['slug'])
     model_files = retrieve_data(source_config['slug'], model_slug, started_at, stopped_at)
     orbits_files = retrieve_data(source_config['slug'], source_config['orbit']['model'], started_at, stopped_at)
+    log.debug("Crunching CSV contents for '%s'..." % source_config['slug'])
     si = StringIO.StringIO()
     cw = csv_writer(si)
     cw.writerow((  # the order matters !
@@ -305,14 +317,16 @@ def generate_csv_contents(source_config, started_at, stopped_at):
                 )
         cdf_handle.close()
+    log.debug("Sorting CSV contents for '%s'..." % source_config['slug'])
     for dkey in sorted(all_data):
         cw.writerow(all_data[dkey])
+    log.info("Done CSV generation for '%s'." % source_config['slug'])
     return si.getvalue()
 def generate_csv_file_if_needed(target_config, started_at, stopped_at):
-    filename = "%s_%s_%s.csv" % (target_config.slug,
+    filename = "%s_%s_%s.csv" % (target_config['slug'],
                                  started_at.strftime(FILE_DATE_FMT),
                                  stopped_at.strftime(FILE_DATE_FMT))
     local_csv_file = get_path("../cache/%s" % filename)
@@ -325,7 +339,7 @@ def generate_csv_file_if_needed(target_config, started_at, stopped_at):
                                               stopped_at=stopped_at))
             log.info("Generation of '%s' done." % filename)
         except Exception as e:
-            abort(500, "Failed creating CSV '%s' : %s" % (filename, e.message))
+            abort(500, "Failed creating CSV '%s' : %s" % (filename, e))
 def increment_hit_counter():
@@ -42,7 +42,7 @@
     SpaceWeather.prototype.init = function(){
       "This is called by the inline bootstrap javascript code.\nThis ain't in the constructor because it might return a Promise later on.\n(for the loader, for example)";
       var started_at, stopped_at, this$ = this;
-      started_at = moment().subtract(2, 'weeks').hours(0).minutes(0).seconds(0);
+      started_at = moment().subtract(1, 'year').hours(0).minutes(0).seconds(0);
       stopped_at = moment().add(1, 'week').hours(0).minutes(0).seconds(0);
       this.setStartAndStop(started_at, stopped_at);
       this.loadAndCreatePlots(started_at, stopped_at);
@@ -105,6 +105,7 @@
         url = sw.buildDataUrlForTarget(target_slug, started_at, stopped_at);
         return d3.csv(url, function(csv){
           var timeFormat, data;
+          console.debug("Requested CSV for " + target_slug + "...", csv);
           timeFormat = d3.timeParse('%Y-%m-%dT%H:%M:%S%Z');
           data = {
             'hci': []
@@ -115,6 +116,9 @@
           if (!csv) {
             reject("CSV is empty or nonexistent at URL '" + url + "'.");
           }
+          if (!csv.length) {
+            reject("CSV is empty at '" + url + "'.");
+          }
           csv.forEach(function(d){
             var dtime;
             dtime = timeFormat(d['time']);
@@ -159,7 +163,7 @@
         targetButton = $(".targets-filters .target." + target.slug);
         targetButton.addClass('loading');
         return this$.loadData(target.slug, started_at, stopped_at).then(function(data){
-          console.info("Loaded CSV data of " + target.name + ".");
+          console.info("Loaded CSV data of " + target.name + ".", data);
           this$.createTimeSeries(target, data);
           this$.orbits.initOrbiter(target.slug, target.config, data['hci']);
           targetButton.removeClass('loading');
@@ -169,7 +173,10 @@
             return this$.disableTarget(target.slug);
           }
         }, function(error){
-          return console.error("Failed loading CSV data of " + target.name + ".", error);
+          console.error("Failed loading CSV data of " + target.name + ".", error);
+          alert("There was an error.\nPlease try again in a few moments.");
+          targetButton.removeClass('loading');
+          return this$.hideLoader();
         });
       });
     };
@@ -74,7 +74,7 @@ https://gitlab.irap.omp.eu/CDPP/SPACEWEATHERONLINE
     """
     # Default time interval is from two weeks ago to one week ahead.
     # We set the h/m/s to zero to benefit from a daily cache.
-    started_at = moment().subtract(2, 'weeks').hours(0).minutes(0).seconds(0)
+    started_at = moment().subtract(1, 'year').hours(0).minutes(0).seconds(0)
     stopped_at = moment().add(1, 'week').hours(0).minutes(0).seconds(0)
     @setStartAndStop(started_at, stopped_at)
     @loadAndCreatePlots(started_at, stopped_at)
@@ -127,12 +127,14 @@ https://gitlab.irap.omp.eu/CDPP/SPACEWEATHERONLINE
     new Promise((resolve, reject) ->
       url = sw.buildDataUrlForTarget(target_slug, started_at, stopped_at)
       d3.csv(url, (csv) ->
+        console.debug("Requested CSV for #{target_slug}...", csv)
         timeFormat = d3.timeParse('%Y-%m-%dT%H:%M:%S%Z')
         data = { 'hci': [] }
         configuration['parameters'].forEach((parameter) ->
           data[parameter['id']] = []
         )
         unless csv then reject "CSV is empty or nonexistent at URL '#{url}'."
+        unless csv.length then reject "CSV is empty at '#{url}'."
         csv.forEach((d) ->
           dtime = timeFormat(d['time'])
           configuration['parameters'].forEach((parameter) ->
@@ -144,7 +146,7 @@ https://gitlab.irap.omp.eu/CDPP/SPACEWEATHERONLINE
               t: dtime, x: parseFloat(d['xhci']), y: parseFloat(d['yhci'])
             })
         )
-        resolve(data)
+        resolve data
       )
     )
@@ -166,13 +168,17 @@ https://gitlab.irap.omp.eu/CDPP/SPACEWEATHERONLINE
       targetButton.addClass('loading')
       @loadData(target.slug, started_at, stopped_at).then(
         (data) ~>
-          console.info "Loaded CSV data of #{target.name}."
+          console.info "Loaded CSV data of #{target.name}.", data
           @createTimeSeries(target, data)
           @orbits.initOrbiter(target.slug, target.config, data['hci'])
           targetButton.removeClass('loading')
           if target.active then @hideLoader() else @disableTarget(target.slug)
         ,
-        (error) -> console.error("Failed loading CSV data of #{target.name}.", error)
+        (error) ~>
+          console.error("Failed loading CSV data of #{target.name}.", error)
+          alert("There was an error.\nPlease try again in a few moments.")
+          targetButton.removeClass('loading')
+          @hideLoader()
       )
     )