From dc0be9925d35db1001545592b3b2bebe31a70421 Mon Sep 17 00:00:00 2001 From: Goutte Date: Mon, 25 Sep 2017 10:29:22 +0200 Subject: [PATCH] Support having no position to display (for Rosetta in some intervals) Make the local cache more resilient to corrupted downloads --- CHANGELOG.md | 7 ++++--- config.yml | 8 ++++---- web/run.py | 14 +++++++++++--- web/static/img/target/rosetta_128.png | Bin 2402 -> 0 bytes web/static/js/swapp.js | 42 ++++++++++++++++++++++++++++++++---------- web/static/js/swapp.ls | 19 ++++++++++++++++--- web/view/home.html.jinja2 | 4 ++-- 7 files changed, 69 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 854441d..38ac848 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,8 +12,8 @@ ## 1.0.0-rc4 - [ ] Make the tarball with netcdf files instead of CSVs -- [ ] Support having no position to display (for Rosetta in some intervals) -- [x] Cache clear (remove all files) `/cache/clear` +- [x] Support having no position to display (for Rosetta in some intervals) +- [x] Make the local cache more resilient to corrupted downloads - [x] Make the local cache more resilient to naming collisions @@ -21,6 +21,7 @@ - [x] Make the targets dynamic in the orbit plot, allowing zoom - [x] Refactor some more to move as much as we can to the config +- [x] Cache clear (remove all files) `/cache/clear` ## 1.0.0-rc2 @@ -65,7 +66,7 @@ - [x] Probes - [x] Rosetta - [x] Juno -- [x] Comets (the one visited by rosetta) +- [x] Comets (the one visited by rosetta, p67) - [x] Orbits axes titles - [x] Logo Europlanet - [x] Cache remote NetCDFs locally diff --git a/config.yml b/config.yml index e99e9f2..20c8806 100644 --- a/config.yml +++ b/config.yml @@ -98,7 +98,7 @@ targets: - type: 'planet' slug: 'earth' name: 'Earth' - title: 'Earth (Coming NEXT!)' + title: 'Earth (coming soon)' orbit: models: - slug: 'earth_orb_all' @@ -152,13 +152,13 @@ targets: - type: 'probe' slug: 'rosetta' name: 'Rosetta' - title: 'Rosetta (coming soon)' + title: 'Rosetta' orbit: models: - slug: 'ros_orb_cruise' models: - slug: 'tao_ros_sw' - locked: true + locked: false default: false - type: 'probe' slug: 'juno' @@ -169,7 +169,7 @@ targets: - slug: 'juno_cruise_all' models: - slug: 'tao_juno_sw' - locked: true + locked: false default: false - type: 'comet' slug: 'p67' diff --git a/web/run.py b/web/run.py index 3c3cd43..f32a957 100755 --- a/web/run.py +++ b/web/run.py @@ -368,11 +368,12 @@ def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at): log.debug("Retrieving '%s'..." % local_gzip_file) urllib.urlretrieve(remote_gzip_file, local_gzip_file) log.debug("Retrieved '%s'." % local_gzip_file) + else: + log.debug("Found '%s' in the cache." % local_gzip_file) local_netc_files = [] for local_gzip_file in local_gzip_files: local_netc_file = local_gzip_file[0:-3] - local_netc_files.append(local_netc_file) log.debug("Unzipping '%s'..." % local_gzip_file) success = True try: @@ -382,9 +383,13 @@ def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at): g.write(file_content) except Exception as e: success = False - log.warning("Cannot process gz file '%s' from '%s' : %s" % - (local_gzip_file, url, e)) + log.error("Cannot process gz file '%s' from '%s' : %s" % + (local_gzip_file, url, e)) + # Sometimes, the downloaded gz is corrupted, and CRC checks fail. + # We want to delete the local gz file and try again next time. + removefile(local_gzip_file) if success: + local_netc_files.append(local_netc_file) log.debug("Unzipped '%s'." % local_gzip_file) return local_netc_files @@ -435,6 +440,8 @@ def get_data_for_target(target_config, started_at, stopped_at): cdf_handle = Dataset(orbit_file, "r", format="NETCDF4") times = cdf_handle.variables['Time'] # YYYY DOY HH MM SS .ms data_hee = cdf_handle.variables['HEE'] + log.debug("%s: aggregating data from '%s'..." % + (target_config['name'], orbit_file)) for time, datum_hee in zip(times, data_hee): dtime = datetime_from_list(time) if started_at <= dtime <= stopped_at: @@ -523,6 +530,7 @@ def generate_csv_file_if_needed(target_slug, started_at, stopped_at): stopped_at=stopped_at)) log.info("Generation of '%s' done." % filename) except Exception as e: + log.error(e) if isfile(local_csv_file): log.warn("Removing failed CSV '%s'..." % local_csv_file) removefile(local_csv_file) diff --git a/web/static/img/target/rosetta_128.png b/web/static/img/target/rosetta_128.png index 8a86492..5e38a91 100644 Binary files a/web/static/img/target/rosetta_128.png and b/web/static/img/target/rosetta_128.png differ diff --git a/web/static/js/swapp.js b/web/static/js/swapp.js index e2d78aa..ba9042d 100644 --- a/web/static/js/swapp.js +++ b/web/static/js/swapp.js @@ -742,7 +742,14 @@ }; Orbits.prototype.initOrbiter = function(slug, config, data){ var orbit_ellipse, orbiter, orbit_line, orbit_section, this$ = this; - console.info("Initializing orbit of " + config.name + "…"); + this.data[slug] = data; + this.orbiters[slug] = config; + if (data.length) { + console.info("Initializing orbit of " + config.name + "…"); + } else { + console.warn("No orbit data for " + config.name + "…"); + return; + } if (slug in this.orbitersElements) { throw new Error("Second init of " + slug); } @@ -755,8 +762,6 @@ return this$.yScale(d.x); }); orbit_section = this.plotWrapper.append('path').datum(data).classed('orbit orbit_section', true); - this.data[slug] = data; - this.orbiters[slug] = config; this.orbitersElements[slug] = { orbiter: orbiter, orbit_ellipse: orbit_ellipse, @@ -771,6 +776,9 @@ return this; }; Orbits.prototype.showOrbiter = function(slug){ + if (!this.data[slug].length) { + return; + } this.orbiters[slug].hidden = false; this.orbitersElements[slug].orbiter.style("display", null); this.orbitersElements[slug].orbit_ellipse.style("display", null); @@ -778,6 +786,9 @@ return this.resize(true); }; Orbits.prototype.hideOrbiter = function(slug){ + if (!this.data[slug].length) { + return; + } this.orbiters[slug].hidden = true; this.orbitersElements[slug].orbiter.style("display", "none"); this.orbitersElements[slug].orbit_ellipse.style("display", "none"); @@ -832,8 +843,12 @@ return this; }; Orbits.prototype.resizeOrbiter = function(slug, config, width, height, animate){ - var tt, el, orbit_section, t, a, b, c, cx, cy, orbit_ellipse; + var data, tt, el, orbit_section, t, a, b, c, cx, cy, orbit_ellipse; animate == null && (animate = false); + data = this.data[slug]; + if (!data.length) { + return; + } console.debug("Resizing orbit of " + slug + "…"); tt = this.svg.transition().duration(750); el = this.orbitersElements[slug]; @@ -861,6 +876,9 @@ var data, el, t; animate == null && (animate = false); data = this.data[slug]; + if (!data.length) { + return; + } datum == null && (datum = this.lastOrbiterData[slug]); datum == null && (datum = data[data.length - 1]); this.lastOrbiterData[slug] = datum; @@ -896,29 +914,33 @@ return this; }; Orbits.prototype.resizeDomain = function(started_at, stopped_at){ - var slug, ref$, config, el, data, results$ = []; + var slug, ref$, config, el, data; for (slug in ref$ = this.orbiters) { config = ref$[slug]; el = this.orbitersElements[slug]; data = this.data[slug].filter(fn$); + if (!data.length) { + return; + } el['orbit_section'].datum(data); - results$.push(el['orbit_section'].attr('d', el['orbit_line'])); + el['orbit_section'].attr('d', el['orbit_line']); } - return results$; function fn$(d){ var ref$; return started_at <= (ref$ = d.t) && ref$ <= stopped_at; } }; Orbits.prototype.resetZoom = function(){ - var slug, ref$, config, el, results$ = []; + var slug, ref$, config, el; for (slug in ref$ = this.orbiters) { config = ref$[slug]; el = this.orbitersElements[slug]; + if (!this.data[slug].length) { + return; + } el['orbit_section'].datum(this.data[slug]); - results$.push(el['orbit_section'].attr('d', el['orbit_line'])); + el['orbit_section'].attr('d', el['orbit_line']); } - return results$; }; return Orbits; }()); diff --git a/web/static/js/swapp.ls b/web/static/js/swapp.ls index f54d999..a335ccc 100644 --- a/web/static/js/swapp.ls +++ b/web/static/js/swapp.ls @@ -700,7 +700,15 @@ export class Orbits @resize() initOrbiter: (slug, config, data) -> - console.info "Initializing orbit of #{config.name}…" + @data[slug] = data + @orbiters[slug] = config + + if data.length + console.info "Initializing orbit of #{config.name}…" + else + console.warn "No orbit data for #{config.name}…" + return + if slug of @orbitersElements then throw new Error("Second init of #{slug}") # The order is important, as it will define the default z-order @@ -719,8 +727,6 @@ export class Orbits .datum(data) .classed('orbit orbit_section', true) - @data[slug] = data - @orbiters[slug] = config @orbitersElements[slug] = orbiter: orbiter orbit_ellipse: orbit_ellipse @@ -737,6 +743,7 @@ export class Orbits this showOrbiter: (slug) -> + if not @data[slug].length then return @orbiters[slug].hidden = false @orbitersElements[slug].orbiter.style("display", null) @orbitersElements[slug].orbit_ellipse.style("display", null) @@ -744,6 +751,7 @@ export class Orbits @resize(true) hideOrbiter: (slug) -> + if not @data[slug].length then return @orbiters[slug].hidden = true @orbitersElements[slug].orbiter.style("display", "none") @orbitersElements[slug].orbit_ellipse.style("display", "none") @@ -797,6 +805,8 @@ export class Orbits this resizeOrbiter: (slug, config, width, height, animate = false) -> + data = @data[slug] + if not data.length then return console.debug("Resizing orbit of #{slug}…") tt = @svg.transition().duration(750) @@ -830,6 +840,7 @@ export class Orbits repositionOrbiter: (slug, datum, animate = false) -> data = @data[slug] + if not data.length then return datum ?= @lastOrbiterData[slug] datum ?= data[data.length - 1] @lastOrbiterData[slug] = datum @@ -859,12 +870,14 @@ export class Orbits for slug, config of @orbiters el = @orbitersElements[slug] data = @data[slug].filter (d) -> started_at <= d.t <= stopped_at + if not data.length then return el['orbit_section'].datum(data) el['orbit_section'].attr('d', el['orbit_line']) resetZoom: -> for slug, config of @orbiters el = @orbitersElements[slug] + if not @data[slug].length then return el['orbit_section'].datum(@data[slug]) el['orbit_section'].attr('d', el['orbit_line']) diff --git a/web/view/home.html.jinja2 b/web/view/home.html.jinja2 index bafa37e..a241858 100755 --- a/web/view/home.html.jinja2 +++ b/web/view/home.html.jinja2 @@ -116,7 +116,7 @@