Support having no position to display (for Rosetta in some intervals)

Make the local cache more resilient to corrupted downloads

Support having no position to display (for Rosetta in some intervals)
Make the local cache more resilient to corrupted downloads
Goutte
1 parent ff3519a8
Showing 7 changed files with 69 additions and 25 deletions Show diff stats
CHANGELOG.md
config.yml
web/run.py
web/static/img/target/rosetta_128.png
web/static/js/swapp.js
web/static/js/swapp.ls
web/view/home.html.jinja2
@@ -12,8 +12,8 @@
 ## 1.0.0-rc4
 - [ ] Make the tarball with netcdf files instead of CSVs
-- [ ] Support having no position to display (for Rosetta in some intervals)
-- [x] Cache clear (remove all files) `/cache/clear`
+- [x] Support having no position to display (for Rosetta in some intervals)
+- [x] Make the local cache more resilient to corrupted downloads
 - [x] Make the local cache more resilient to naming collisions
@@ -21,6 +21,7 @@
 - [x] Make the targets dynamic in the orbit plot, allowing zoom
 - [x] Refactor some more to move as much as we can to the config
+- [x] Cache clear (remove all files) `/cache/clear`
 ## 1.0.0-rc2
@@ -65,7 +66,7 @@
 - [x] Probes
     - [x] Rosetta
     - [x] Juno
-- [x] Comets (the one visited by rosetta)
+- [x] Comets (the one visited by rosetta, p67)
 - [x] Orbits axes titles
 - [x] Logo Europlanet
 - [x] Cache remote NetCDFs locally
@@ -98,7 +98,7 @@ targets:
     - type: 'planet'
       slug: 'earth'
       name: 'Earth'
-      title: 'Earth (Coming NEXT!)'
+      title: 'Earth (coming soon)'
       orbit:
         models:
           - slug: 'earth_orb_all'
@@ -152,13 +152,13 @@ targets:
     - type: 'probe'
       slug: 'rosetta'
       name: 'Rosetta'
-      title: 'Rosetta (coming soon)'
+      title: 'Rosetta'
       orbit:
         models:
           - slug: 'ros_orb_cruise'
       models:
           - slug: 'tao_ros_sw'
-      locked: true
+      locked: false
       default: false
     - type: 'probe'
       slug: 'juno'
@@ -169,7 +169,7 @@ targets:
           - slug: 'juno_cruise_all'
       models:
           - slug: 'tao_juno_sw'
-      locked: true
+      locked: false
       default: false
     - type: 'comet'
       slug: 'p67'
@@ -368,11 +368,12 @@ def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at):
             log.debug("Retrieving '%s'..." % local_gzip_file)
             urllib.urlretrieve(remote_gzip_file, local_gzip_file)
             log.debug("Retrieved '%s'." % local_gzip_file)
+        else:
+            log.debug("Found '%s' in the cache." % local_gzip_file)
     local_netc_files = []
     for local_gzip_file in local_gzip_files:
         local_netc_file = local_gzip_file[0:-3]
-        local_netc_files.append(local_netc_file)
         log.debug("Unzipping '%s'..." % local_gzip_file)
         success = True
         try:
@@ -382,9 +383,13 @@ def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at):
                     g.write(file_content)
         except Exception as e:
             success = False
-            log.warning("Cannot process gz file '%s' from '%s' : %s" %
-                        (local_gzip_file, url, e))
+            log.error("Cannot process gz file '%s' from '%s' : %s" %
+                      (local_gzip_file, url, e))
+            # Sometimes, the downloaded gz is corrupted, and CRC checks fail.
+            # We want to delete the local gz file and try again next time.
+            removefile(local_gzip_file)
         if success:
+            local_netc_files.append(local_netc_file)
             log.debug("Unzipped '%s'." % local_gzip_file)
     return local_netc_files
@@ -435,6 +440,8 @@ def get_data_for_target(target_config, started_at, stopped_at):
         cdf_handle = Dataset(orbit_file, "r", format="NETCDF4")
         times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
         data_hee = cdf_handle.variables['HEE']
+        log.debug("%s: aggregating data from '%s'..." %
+                  (target_config['name'], orbit_file))
         for time, datum_hee in zip(times, data_hee):
             dtime = datetime_from_list(time)
             if started_at <= dtime <= stopped_at:
@@ -523,6 +530,7 @@ def generate_csv_file_if_needed(target_slug, started_at, stopped_at):
                                               stopped_at=stopped_at))
             log.info("Generation of '%s' done." % filename)
         except Exception as e:
+            log.error(e)
             if isfile(local_csv_file):
                 log.warn("Removing failed CSV '%s'..." % local_csv_file)
                 removefile(local_csv_file)
@@ -742,7 +742,14 @@
     };
     Orbits.prototype.initOrbiter = function(slug, config, data){
       var orbit_ellipse, orbiter, orbit_line, orbit_section, this$ = this;
-      console.info("Initializing orbit of " + config.name + "…");
+      this.data[slug] = data;
+      this.orbiters[slug] = config;
+      if (data.length) {
+        console.info("Initializing orbit of " + config.name + "…");
+      } else {
+        console.warn("No orbit data for " + config.name + "…");
+        return;
+      }
       if (slug in this.orbitersElements) {
         throw new Error("Second init of " + slug);
       }
@@ -755,8 +762,6 @@
         return this$.yScale(d.x);
       });
       orbit_section = this.plotWrapper.append('path').datum(data).classed('orbit orbit_section', true);
-      this.data[slug] = data;
-      this.orbiters[slug] = config;
       this.orbitersElements[slug] = {
         orbiter: orbiter,
         orbit_ellipse: orbit_ellipse,
@@ -771,6 +776,9 @@
       return this;
     };
     Orbits.prototype.showOrbiter = function(slug){
+      if (!this.data[slug].length) {
+        return;
+      }
       this.orbiters[slug].hidden = false;
       this.orbitersElements[slug].orbiter.style("display", null);
       this.orbitersElements[slug].orbit_ellipse.style("display", null);
@@ -778,6 +786,9 @@
       return this.resize(true);
     };
     Orbits.prototype.hideOrbiter = function(slug){
+      if (!this.data[slug].length) {
+        return;
+      }
       this.orbiters[slug].hidden = true;
       this.orbitersElements[slug].orbiter.style("display", "none");
       this.orbitersElements[slug].orbit_ellipse.style("display", "none");
@@ -832,8 +843,12 @@
       return this;
     };
     Orbits.prototype.resizeOrbiter = function(slug, config, width, height, animate){
-      var tt, el, orbit_section, t, a, b, c, cx, cy, orbit_ellipse;
+      var data, tt, el, orbit_section, t, a, b, c, cx, cy, orbit_ellipse;
       animate == null && (animate = false);
+      data = this.data[slug];
+      if (!data.length) {
+        return;
+      }
       console.debug("Resizing orbit of " + slug + "…");
       tt = this.svg.transition().duration(750);
       el = this.orbitersElements[slug];
@@ -861,6 +876,9 @@
       var data, el, t;
       animate == null && (animate = false);
       data = this.data[slug];
+      if (!data.length) {
+        return;
+      }
       datum == null && (datum = this.lastOrbiterData[slug]);
       datum == null && (datum = data[data.length - 1]);
       this.lastOrbiterData[slug] = datum;
@@ -896,29 +914,33 @@
       return this;
     };
     Orbits.prototype.resizeDomain = function(started_at, stopped_at){
-      var slug, ref$, config, el, data, results$ = [];
+      var slug, ref$, config, el, data;
       for (slug in ref$ = this.orbiters) {
         config = ref$[slug];
         el = this.orbitersElements[slug];
         data = this.data[slug].filter(fn$);
+        if (!data.length) {
+          return;
+        }
         el['orbit_section'].datum(data);
-        results$.push(el['orbit_section'].attr('d', el['orbit_line']));
+        el['orbit_section'].attr('d', el['orbit_line']);
       }
-      return results$;
       function fn$(d){
         var ref$;
         return started_at <= (ref$ = d.t) && ref$ <= stopped_at;
       }
     };
     Orbits.prototype.resetZoom = function(){
-      var slug, ref$, config, el, results$ = [];
+      var slug, ref$, config, el;
       for (slug in ref$ = this.orbiters) {
         config = ref$[slug];
         el = this.orbitersElements[slug];
+        if (!this.data[slug].length) {
+          return;
+        }
         el['orbit_section'].datum(this.data[slug]);
-        results$.push(el['orbit_section'].attr('d', el['orbit_line']));
+        el['orbit_section'].attr('d', el['orbit_line']);
       }
-      return results$;
     };
     return Orbits;
   }());
@@ -700,7 +700,15 @@ export class Orbits
     @resize()
   initOrbiter: (slug, config, data) ->
-    console.info "Initializing orbit of #{config.name}…"
+    @data[slug] = data
+    @orbiters[slug] = config
+
+    if data.length
+      console.info "Initializing orbit of #{config.name}…"
+    else
+      console.warn "No orbit data for #{config.name}…"
+      return
+
     if slug of @orbitersElements then throw new Error("Second init of #{slug}")
     # The order is important, as it will define the default z-order
@@ -719,8 +727,6 @@ export class Orbits
                                 .datum(data)
                                 .classed('orbit orbit_section', true)
-    @data[slug] = data
-    @orbiters[slug] = config
     @orbitersElements[slug] =
       orbiter: orbiter
       orbit_ellipse: orbit_ellipse
@@ -737,6 +743,7 @@ export class Orbits
     this
   showOrbiter: (slug) ->
+    if not @data[slug].length then return
     @orbiters[slug].hidden = false
     @orbitersElements[slug].orbiter.style("display", null)
     @orbitersElements[slug].orbit_ellipse.style("display", null)
@@ -744,6 +751,7 @@ export class Orbits
     @resize(true)
   hideOrbiter: (slug) ->
+    if not @data[slug].length then return
     @orbiters[slug].hidden = true
     @orbitersElements[slug].orbiter.style("display", "none")
     @orbitersElements[slug].orbit_ellipse.style("display", "none")
@@ -797,6 +805,8 @@ export class Orbits
     this
   resizeOrbiter: (slug, config, width, height, animate = false) ->
+    data = @data[slug]
+    if not data.length then return
     console.debug("Resizing orbit of #{slug}…")
     tt = @svg.transition().duration(750)
@@ -830,6 +840,7 @@ export class Orbits
   repositionOrbiter: (slug, datum, animate = false) ->
     data = @data[slug]
+    if not data.length then return
     datum ?= @lastOrbiterData[slug]
     datum ?= data[data.length - 1]
     @lastOrbiterData[slug] = datum
@@ -859,12 +870,14 @@ export class Orbits
     for slug, config of @orbiters
       el = @orbitersElements[slug]
       data = @data[slug].filter (d) -> started_at <= d.t <= stopped_at
+      if not data.length then return
       el['orbit_section'].datum(data)
       el['orbit_section'].attr('d', el['orbit_line'])
   resetZoom: ->
     for slug, config of @orbiters
       el = @orbitersElements[slug]
+      if not @data[slug].length then return
       el['orbit_section'].datum(@data[slug])
       el['orbit_section'].attr('d', el['orbit_line'])
@@ -116,7 +116,7 @@
         <section id="orbits"></section>
         <div class="plots-actions plots-buttons">
           <button id="download" class="mdl-button mdl-button--raised mdl-button--primary"
-                  title="Download the CSV raw data for each target in a tarball.">
+                  title="Download the raw data for each selected target and parameters.">
             Download
           </button>
           <button id="samp" class="samp mdl-button mdl-button--raised mdl-button--primary mdl-button--disabled"
@@ -535,7 +535,7 @@ var configuration = {
   orbits_container: '#orbits',
   api: {
       'data_for_interval': "{{ request.url_root }}<target>_<started_at>_<stopped_at>.csv",
-      'download': "{{ request.url_root }}<targets>_<started_at>_<stopped_at>.tar.gz",
+      'download': "{{ request.url_root }}<targets>_<started_at>_<stopped_at>.nc",
       'samp': "{{ request.url_root }}<targets>_<params>_<started_at>_<stopped_at>.nc"
   },
   sun: {