CDPP / HELIOPROPA

Commit 9bfa6c42ebd36db7c2de22354ef26ac90097e50e

Authored by Goutte
1 parent 97d6cb96
Exists in master and in 2 other branches DEV, rhitier-dev

More bug hunting.

Showing 3 changed files with 37 additions and 10 deletions   Show diff stats
Inline Side-by-side
web/run.py
  Diff comments   View file @9bfa6c4
... ... @@ -47,7 +47,7 @@ FILE_DATE_FMT = "%Y-%m-%dT%H:%M:%S"
47 47 # LOGGING #####################################################################
48 48  
49 49 log = logging.getLogger("HelioPropa")
50   -log.setLevel(logging.INFO)
  50 +log.setLevel(logging.DEBUG)
51 51 logHandler = logging.FileHandler(get_path('run.log'))
52 52 logHandler.setFormatter(logging.Formatter(
53 53 "%(asctime)s - %(levelname)s - %(message)s"
... ... @@ -214,10 +214,16 @@ def retrieve_data(orbiter, what, started_at, stopped_at):
214 214 abort(400, "Failed to fetch data at '%s'." % url)
215 215 if remote_gzip_files == 'NODATASET':
216 216 abort(400, "No dataset at '%s'." % url)
  217 + # if remote_gzip_files == ['OUTOFTIME']:
  218 + # abort(400, "No data for time interval at '%s'." % url)
  219 +
  220 + log.debug("Fetched remote gzip files list : %s." % str(remote_gzip_files))
217 221  
218 222 # retriever = urllib.URLopener() # would we need to do this every time ?
219 223 local_gzip_files = []
220 224 for remote_gzip_file in remote_gzip_files:
  225 + if remote_gzip_file == 'OUTOFTIME':
  226 + continue
221 227 # hotfix removeme @Myriam
222 228 if remote_gzip_file.endswith('/.gz'):
223 229 continue
... ... @@ -227,16 +233,20 @@ def retrieve_data(orbiter, what, started_at, stopped_at):
227 233 local_gzip_file = get_path("../cache/%s" % filename)
228 234 local_gzip_files.append(local_gzip_file)
229 235 if not isfile(local_gzip_file):
  236 + log.debug("Retrieving '%s'..." % local_gzip_file)
230 237 urllib.urlretrieve(remote_gzip_file, local_gzip_file)
  238 + log.debug("Retrieved '%s'." % local_gzip_file)
231 239  
232 240 local_netc_files = []
233 241 for local_gzip_file in local_gzip_files:
234 242 local_netc_file = local_gzip_file[0:-3]
235 243 local_netc_files.append(local_netc_file)
  244 + log.debug("Unzipping '%s'..." % local_gzip_file)
236 245 with gzip.open(local_gzip_file, 'rb') as f:
237 246 file_content = f.read()
238 247 with open(local_netc_file, 'w+b') as g:
239 248 g.write(file_content)
  249 + log.debug("Unzipped '%s'." % local_gzip_file)
240 250  
241 251 return local_netc_files
242 252  
... ... @@ -251,9 +261,11 @@ def generate_csv_contents(source_config, started_at, stopped_at):
251 261 # Grab the list of netCDF files from Myriam's API
252 262 # http://cdpp.irap.omp.eu/BASE/DDService/getDataUrl.php?dataSet=jupiter_orb_all&StartTime=2014-02-23T10:00:10&StopTime=2017-02-24T23:59:00
253 263 # http://cdpp.irap.omp.eu/BASE/DATA/TAO/JUPITER/SW/sw_2014.nc.gz
  264 + log.info("Generating CSV for '%s'..." % source_config['slug'])
254 265 model_files = retrieve_data(source_config['slug'], model_slug, started_at, stopped_at)
255 266 orbits_files = retrieve_data(source_config['slug'], source_config['orbit']['model'], started_at, stopped_at)
256 267  
  268 + log.debug("Crunching CSV contents for '%s'..." % source_config['slug'])
257 269 si = StringIO.StringIO()
258 270 cw = csv_writer(si)
259 271 cw.writerow(( # the order matters !
... ... @@ -305,14 +317,16 @@ def generate_csv_contents(source_config, started_at, stopped_at):
305 317 )
306 318 cdf_handle.close()
307 319  
  320 + log.debug("Sorting CSV contents for '%s'..." % source_config['slug'])
308 321 for dkey in sorted(all_data):
309 322 cw.writerow(all_data[dkey])
310 323  
  324 + log.info("Done CSV generation for '%s'." % source_config['slug'])
311 325 return si.getvalue()
312 326  
313 327  
314 328 def generate_csv_file_if_needed(target_config, started_at, stopped_at):
315   - filename = "%s_%s_%s.csv" % (target_config.slug,
  329 + filename = "%s_%s_%s.csv" % (target_config['slug'],
316 330 started_at.strftime(FILE_DATE_FMT),
317 331 stopped_at.strftime(FILE_DATE_FMT))
318 332 local_csv_file = get_path("../cache/%s" % filename)
... ... @@ -325,7 +339,7 @@ def generate_csv_file_if_needed(target_config, started_at, stopped_at):
325 339 stopped_at=stopped_at))
326 340 log.info("Generation of '%s' done." % filename)
327 341 except Exception as e:
328   - abort(500, "Failed creating CSV '%s' : %s" % (filename, e.message))
  342 + abort(500, "Failed creating CSV '%s' : %s" % (filename, e))
329 343  
330 344  
331 345 def increment_hit_counter():
... ...
web/static/js/swapp.js
  Diff comments   View file @9bfa6c4
... ... @@ -42,7 +42,7 @@
42 42 SpaceWeather.prototype.init = function(){
43 43 "This is called by the inline bootstrap javascript code.\nThis ain't in the constructor because it might return a Promise later on.\n(for the loader, for example)";
44 44 var started_at, stopped_at, this$ = this;
45   - started_at = moment().subtract(2, 'weeks').hours(0).minutes(0).seconds(0);
  45 + started_at = moment().subtract(1, 'year').hours(0).minutes(0).seconds(0);
46 46 stopped_at = moment().add(1, 'week').hours(0).minutes(0).seconds(0);
47 47 this.setStartAndStop(started_at, stopped_at);
48 48 this.loadAndCreatePlots(started_at, stopped_at);
... ... @@ -105,6 +105,7 @@
105 105 url = sw.buildDataUrlForTarget(target_slug, started_at, stopped_at);
106 106 return d3.csv(url, function(csv){
107 107 var timeFormat, data;
  108 + console.debug("Requested CSV for " + target_slug + "...", csv);
108 109 timeFormat = d3.timeParse('%Y-%m-%dT%H:%M:%S%Z');
109 110 data = {
110 111 'hci': []
... ... @@ -115,6 +116,9 @@
115 116 if (!csv) {
116 117 reject("CSV is empty or nonexistent at URL '" + url + "'.");
117 118 }
  119 + if (!csv.length) {
  120 + reject("CSV is empty at '" + url + "'.");
  121 + }
118 122 csv.forEach(function(d){
119 123 var dtime;
120 124 dtime = timeFormat(d['time']);
... ... @@ -159,7 +163,7 @@
159 163 targetButton = $(".targets-filters .target." + target.slug);
160 164 targetButton.addClass('loading');
161 165 return this$.loadData(target.slug, started_at, stopped_at).then(function(data){
162   - console.info("Loaded CSV data of " + target.name + ".");
  166 + console.info("Loaded CSV data of " + target.name + ".", data);
163 167 this$.createTimeSeries(target, data);
164 168 this$.orbits.initOrbiter(target.slug, target.config, data['hci']);
165 169 targetButton.removeClass('loading');
... ... @@ -169,7 +173,10 @@
169 173 return this$.disableTarget(target.slug);
170 174 }
171 175 }, function(error){
172   - return console.error("Failed loading CSV data of " + target.name + ".", error);
  176 + console.error("Failed loading CSV data of " + target.name + ".", error);
  177 + alert("There was an error.\nPlease try again in a few moments.");
  178 + targetButton.removeClass('loading');
  179 + return this$.hideLoader();
173 180 });
174 181 });
175 182 };
... ...
web/static/js/swapp.ls
  Diff comments   View file @9bfa6c4
... ... @@ -74,7 +74,7 @@ https://gitlab.irap.omp.eu/CDPP/SPACEWEATHERONLINE
74 74 """
75 75 # Default time interval is from two weeks ago to one week ahead.
76 76 # We set the h/m/s to zero to benefit from a daily cache.
77   - started_at = moment().subtract(2, 'weeks').hours(0).minutes(0).seconds(0)
  77 + started_at = moment().subtract(1, 'year').hours(0).minutes(0).seconds(0)
78 78 stopped_at = moment().add(1, 'week').hours(0).minutes(0).seconds(0)
79 79 @setStartAndStop(started_at, stopped_at)
80 80 @loadAndCreatePlots(started_at, stopped_at)
... ... @@ -127,12 +127,14 @@ https://gitlab.irap.omp.eu/CDPP/SPACEWEATHERONLINE
127 127 new Promise((resolve, reject) ->
128 128 url = sw.buildDataUrlForTarget(target_slug, started_at, stopped_at)
129 129 d3.csv(url, (csv) ->
  130 + console.debug("Requested CSV for #{target_slug}...", csv)
130 131 timeFormat = d3.timeParse('%Y-%m-%dT%H:%M:%S%Z')
131 132 data = { 'hci': [] }
132 133 configuration['parameters'].forEach((parameter) ->
133 134 data[parameter['id']] = []
134 135 )
135 136 unless csv then reject "CSV is empty or nonexistent at URL '#{url}'."
  137 + unless csv.length then reject "CSV is empty at '#{url}'."
136 138 csv.forEach((d) ->
137 139 dtime = timeFormat(d['time'])
138 140 configuration['parameters'].forEach((parameter) ->
... ... @@ -144,7 +146,7 @@ https://gitlab.irap.omp.eu/CDPP/SPACEWEATHERONLINE
144 146 t: dtime, x: parseFloat(d['xhci']), y: parseFloat(d['yhci'])
145 147 })
146 148 )
147   - resolve(data)
  149 + resolve data
148 150 )
149 151 )
150 152  
... ... @@ -166,13 +168,17 @@ https://gitlab.irap.omp.eu/CDPP/SPACEWEATHERONLINE
166 168 targetButton.addClass('loading')
167 169 @loadData(target.slug, started_at, stopped_at).then(
168 170 (data) ~>
169   - console.info "Loaded CSV data of #{target.name}."
  171 + console.info "Loaded CSV data of #{target.name}.", data
170 172 @createTimeSeries(target, data)
171 173 @orbits.initOrbiter(target.slug, target.config, data['hci'])
172 174 targetButton.removeClass('loading')
173 175 if target.active then @hideLoader() else @disableTarget(target.slug)
174 176 ,
175   - (error) -> console.error("Failed loading CSV data of #{target.name}.", error)
  177 + (error) ~>
  178 + console.error("Failed loading CSV data of #{target.name}.", error)
  179 + alert("There was an error.\nPlease try again in a few moments.")
  180 + targetButton.removeClass('loading')
  181 + @hideLoader()
176 182 )
177 183 )
178 184  
... ...