From 9bfa6c42ebd36db7c2de22354ef26ac90097e50e Mon Sep 17 00:00:00 2001 From: Goutte Date: Thu, 20 Jul 2017 12:41:19 +0200 Subject: [PATCH] More bug hunting. --- web/run.py | 20 +++++++++++++++++--- web/static/js/swapp.js | 13 ++++++++++--- web/static/js/swapp.ls | 14 ++++++++++---- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/web/run.py b/web/run.py index 5352d9d..02f37bc 100755 --- a/web/run.py +++ b/web/run.py @@ -47,7 +47,7 @@ FILE_DATE_FMT = "%Y-%m-%dT%H:%M:%S" # LOGGING ##################################################################### log = logging.getLogger("HelioPropa") -log.setLevel(logging.INFO) +log.setLevel(logging.DEBUG) logHandler = logging.FileHandler(get_path('run.log')) logHandler.setFormatter(logging.Formatter( "%(asctime)s - %(levelname)s - %(message)s" @@ -214,10 +214,16 @@ def retrieve_data(orbiter, what, started_at, stopped_at): abort(400, "Failed to fetch data at '%s'." % url) if remote_gzip_files == 'NODATASET': abort(400, "No dataset at '%s'." % url) + # if remote_gzip_files == ['OUTOFTIME']: + # abort(400, "No data for time interval at '%s'." % url) + + log.debug("Fetched remote gzip files list : %s." % str(remote_gzip_files)) # retriever = urllib.URLopener() # would we need to do this every time ? local_gzip_files = [] for remote_gzip_file in remote_gzip_files: + if remote_gzip_file == 'OUTOFTIME': + continue # hotfix removeme @Myriam if remote_gzip_file.endswith('/.gz'): continue @@ -227,16 +233,20 @@ def retrieve_data(orbiter, what, started_at, stopped_at): local_gzip_file = get_path("../cache/%s" % filename) local_gzip_files.append(local_gzip_file) if not isfile(local_gzip_file): + log.debug("Retrieving '%s'..." % local_gzip_file) urllib.urlretrieve(remote_gzip_file, local_gzip_file) + log.debug("Retrieved '%s'." % local_gzip_file) local_netc_files = [] for local_gzip_file in local_gzip_files: local_netc_file = local_gzip_file[0:-3] local_netc_files.append(local_netc_file) + log.debug("Unzipping '%s'..." % local_gzip_file) with gzip.open(local_gzip_file, 'rb') as f: file_content = f.read() with open(local_netc_file, 'w+b') as g: g.write(file_content) + log.debug("Unzipped '%s'." % local_gzip_file) return local_netc_files @@ -251,9 +261,11 @@ def generate_csv_contents(source_config, started_at, stopped_at): # Grab the list of netCDF files from Myriam's API # http://cdpp.irap.omp.eu/BASE/DDService/getDataUrl.php?dataSet=jupiter_orb_all&StartTime=2014-02-23T10:00:10&StopTime=2017-02-24T23:59:00 # http://cdpp.irap.omp.eu/BASE/DATA/TAO/JUPITER/SW/sw_2014.nc.gz + log.info("Generating CSV for '%s'..." % source_config['slug']) model_files = retrieve_data(source_config['slug'], model_slug, started_at, stopped_at) orbits_files = retrieve_data(source_config['slug'], source_config['orbit']['model'], started_at, stopped_at) + log.debug("Crunching CSV contents for '%s'..." % source_config['slug']) si = StringIO.StringIO() cw = csv_writer(si) cw.writerow(( # the order matters ! @@ -305,14 +317,16 @@ def generate_csv_contents(source_config, started_at, stopped_at): ) cdf_handle.close() + log.debug("Sorting CSV contents for '%s'..." % source_config['slug']) for dkey in sorted(all_data): cw.writerow(all_data[dkey]) + log.info("Done CSV generation for '%s'." % source_config['slug']) return si.getvalue() def generate_csv_file_if_needed(target_config, started_at, stopped_at): - filename = "%s_%s_%s.csv" % (target_config.slug, + filename = "%s_%s_%s.csv" % (target_config['slug'], started_at.strftime(FILE_DATE_FMT), stopped_at.strftime(FILE_DATE_FMT)) local_csv_file = get_path("../cache/%s" % filename) @@ -325,7 +339,7 @@ def generate_csv_file_if_needed(target_config, started_at, stopped_at): stopped_at=stopped_at)) log.info("Generation of '%s' done." % filename) except Exception as e: - abort(500, "Failed creating CSV '%s' : %s" % (filename, e.message)) + abort(500, "Failed creating CSV '%s' : %s" % (filename, e)) def increment_hit_counter(): diff --git a/web/static/js/swapp.js b/web/static/js/swapp.js index fce2e3c..5e00474 100644 --- a/web/static/js/swapp.js +++ b/web/static/js/swapp.js @@ -42,7 +42,7 @@ SpaceWeather.prototype.init = function(){ "This is called by the inline bootstrap javascript code.\nThis ain't in the constructor because it might return a Promise later on.\n(for the loader, for example)"; var started_at, stopped_at, this$ = this; - started_at = moment().subtract(2, 'weeks').hours(0).minutes(0).seconds(0); + started_at = moment().subtract(1, 'year').hours(0).minutes(0).seconds(0); stopped_at = moment().add(1, 'week').hours(0).minutes(0).seconds(0); this.setStartAndStop(started_at, stopped_at); this.loadAndCreatePlots(started_at, stopped_at); @@ -105,6 +105,7 @@ url = sw.buildDataUrlForTarget(target_slug, started_at, stopped_at); return d3.csv(url, function(csv){ var timeFormat, data; + console.debug("Requested CSV for " + target_slug + "...", csv); timeFormat = d3.timeParse('%Y-%m-%dT%H:%M:%S%Z'); data = { 'hci': [] @@ -115,6 +116,9 @@ if (!csv) { reject("CSV is empty or nonexistent at URL '" + url + "'."); } + if (!csv.length) { + reject("CSV is empty at '" + url + "'."); + } csv.forEach(function(d){ var dtime; dtime = timeFormat(d['time']); @@ -159,7 +163,7 @@ targetButton = $(".targets-filters .target." + target.slug); targetButton.addClass('loading'); return this$.loadData(target.slug, started_at, stopped_at).then(function(data){ - console.info("Loaded CSV data of " + target.name + "."); + console.info("Loaded CSV data of " + target.name + ".", data); this$.createTimeSeries(target, data); this$.orbits.initOrbiter(target.slug, target.config, data['hci']); targetButton.removeClass('loading'); @@ -169,7 +173,10 @@ return this$.disableTarget(target.slug); } }, function(error){ - return console.error("Failed loading CSV data of " + target.name + ".", error); + console.error("Failed loading CSV data of " + target.name + ".", error); + alert("There was an error.\nPlease try again in a few moments."); + targetButton.removeClass('loading'); + return this$.hideLoader(); }); }); }; diff --git a/web/static/js/swapp.ls b/web/static/js/swapp.ls index cb6bca1..5477d76 100644 --- a/web/static/js/swapp.ls +++ b/web/static/js/swapp.ls @@ -74,7 +74,7 @@ https://gitlab.irap.omp.eu/CDPP/SPACEWEATHERONLINE """ # Default time interval is from two weeks ago to one week ahead. # We set the h/m/s to zero to benefit from a daily cache. - started_at = moment().subtract(2, 'weeks').hours(0).minutes(0).seconds(0) + started_at = moment().subtract(1, 'year').hours(0).minutes(0).seconds(0) stopped_at = moment().add(1, 'week').hours(0).minutes(0).seconds(0) @setStartAndStop(started_at, stopped_at) @loadAndCreatePlots(started_at, stopped_at) @@ -127,12 +127,14 @@ https://gitlab.irap.omp.eu/CDPP/SPACEWEATHERONLINE new Promise((resolve, reject) -> url = sw.buildDataUrlForTarget(target_slug, started_at, stopped_at) d3.csv(url, (csv) -> + console.debug("Requested CSV for #{target_slug}...", csv) timeFormat = d3.timeParse('%Y-%m-%dT%H:%M:%S%Z') data = { 'hci': [] } configuration['parameters'].forEach((parameter) -> data[parameter['id']] = [] ) unless csv then reject "CSV is empty or nonexistent at URL '#{url}'." + unless csv.length then reject "CSV is empty at '#{url}'." csv.forEach((d) -> dtime = timeFormat(d['time']) configuration['parameters'].forEach((parameter) -> @@ -144,7 +146,7 @@ https://gitlab.irap.omp.eu/CDPP/SPACEWEATHERONLINE t: dtime, x: parseFloat(d['xhci']), y: parseFloat(d['yhci']) }) ) - resolve(data) + resolve data ) ) @@ -166,13 +168,17 @@ https://gitlab.irap.omp.eu/CDPP/SPACEWEATHERONLINE targetButton.addClass('loading') @loadData(target.slug, started_at, stopped_at).then( (data) ~> - console.info "Loaded CSV data of #{target.name}." + console.info "Loaded CSV data of #{target.name}.", data @createTimeSeries(target, data) @orbits.initOrbiter(target.slug, target.config, data['hci']) targetButton.removeClass('loading') if target.active then @hideLoader() else @disableTarget(target.slug) , - (error) -> console.error("Failed loading CSV data of #{target.name}.", error) + (error) ~> + console.error("Failed loading CSV data of #{target.name}.", error) + alert("There was an error.\nPlease try again in a few moments.") + targetButton.removeClass('loading') + @hideLoader() ) ) -- libgit2 0.21.2