Cache generated CSVs, and serve them as static files so that clients can cache t…

…hem too. Implement a visits counter.

Cache generated CSVs, and serve them as static files so that clients can cache t…
…hem too. Implement a visits counter.
Goutte
1 parent 79a09c02
Showing 6 changed files with 237 additions and 62 deletions Show diff stats
.gitignore
CHANGELOG.md
web/run.py
web/static/js/swapp.js
web/static/js/swapp.ls
web/view/home.html.jinja2
 # Local configuration
 secret.yml
  
+# Local databases
+cache/*
+VISITS
+
 # Dolphin
 .directory
  
 ## TODO
  
-- Support enabling/disabling sources
-- Start/Stop datetime fields
-- Cache generated CSVs locally
-- Download raw data (as CSV) for current time interval
-- Same via SAMP
+- Support multiple models for each target
 - Play button to start time dimension
  
+## 1.0.0
+
+- [x] Cache generated CSVs on the server side
+- [x] Cache generated CSVs on the client side
+- [x] Visits counter
+- [ ] Start/Stop datetime fields
+- [ ] Zoom in on time series
+- [ ] Loader
+- [ ] Download raw data (as CSV) for current time interval and targets
+- [ ] Same via SAMP
+
+
 ## 0.0.0
  
 - Initial website skeleton
@@ -26,3 +34,4 @@
 - Cache remote NetCDFs locally
 - Add source name on time series
 - Support multiple sources
+- Support enabling/disabling sources
@@ -27,6 +27,7 @@ THIS_DIRECTORY = dirname(abspath(__file__))
  
  
 def get_path(relative_path):
+    """Get an absolute path from the relative path to this script directory."""
     return abspath(join(THIS_DIRECTORY, relative_path))
  
  
@@ -209,60 +210,18 @@ def retrieve_data(orbiter, what, started_at, stopped_at):
     return local_netc_files
  
  
-# ROUTING #####################################################################
-
-@app.route('/favicon.ico')
-def favicon():
-    return send_from_directory(
-        join(app.root_path, 'static', 'img'),
-        'favicon.ico', mimetype='image/vnd.microsoft.icon'
-    )
-
-
-@app.route("/")
-@app.route("/home.html")
-@app.route("/index.html")
-def home():
-    return render_view('home.html.jinja2', {
-        'targets': config['targets'],
-        'planets': [s for s in config['targets'] if s['type'] == 'planet'],
-        'probes':  [s for s in config['targets'] if s['type'] == 'probe'],
-        'comets':  [s for s in config['targets'] if s['type'] == 'comet'],
-    })
-
-
-@app.route("/<source>/data.csv")
-def get_orbiter_csv(source):
-    """
-    Grab data and orbit data for the specified `source`,
-    rearrange it and return it as a CSV file.
-    """
-    # http://cdpp1.cesr.fr/BASE/DDService/getDataUrl.php?dataSet=tao_ros_sw&StartTime=2014-02-23T10:00&StopTime=2016-02-24T23:59
-    # Process input parameters
-    source_config = get_source_config(source)
-    date_fmt = "%Y-%m-%dT%H:%M:%S"
-    started_at = request.args.get('started_at')
-    try:
-        started_at = datetime.datetime.strptime(started_at, date_fmt)
-    except:
-        abort(400, "Invalid started_at parameter : '%s'." % started_at)
-    stopped_at = request.args.get('stopped_at')
-    try:
-        stopped_at = datetime.datetime.strptime(stopped_at, date_fmt)
-    except:
-        abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at)
-
+def generate_csv_contents(source_config, started_at, stopped_at):
     # todo: iterate on models when there are many
     try:
         model_slug = source_config['models'][0]['slug']
     except:
-        abort(500, "Invalid model configuration for '%s'." % source)
+        abort(500, "Invalid model configuration for '%s'." % source_config['slug'])
  
     # Grab the list of netCDF files from Myriam's API
     # http://cdpp.irap.omp.eu/BASE/DDService/getDataUrl.php?dataSet=jupiter_orb_all&StartTime=2014-02-23T10:00:10&StopTime=2017-02-24T23:59:00
     # http://cdpp.irap.omp.eu/BASE/DATA/TAO/JUPITER/SW/sw_2014.nc.gz
-    model_files = retrieve_data(source, model_slug, started_at, stopped_at)
-    orbits_files = retrieve_data(source, source_config['orbit']['model'], started_at, stopped_at)
+    model_files = retrieve_data(source_config['slug'], model_slug, started_at, stopped_at)
+    orbits_files = retrieve_data(source_config['slug'], source_config['orbit']['model'], started_at, stopped_at)
  
     si = StringIO.StringIO()
     cw = csv_writer(si)
@@ -321,6 +280,179 @@ def get_orbiter_csv(source):
     return si.getvalue()
  
  
+def increment_hit_counter():
+    hit_count_path = get_path("../VISITS")
+
+    if isfile(hit_count_path):
+        hit_count = int(open(hit_count_path).read())
+        hit_count += 1
+    else:
+        hit_count = 1
+
+    hit_counter_file = open(hit_count_path, 'w')
+    hit_counter_file.write(str(hit_count))
+    hit_counter_file.close()
+
+    return hit_count
+
+
+# ROUTING #####################################################################
+
+@app.route('/favicon.ico')
+def favicon():
+    return send_from_directory(
+        join(app.root_path, 'static', 'img'),
+        'favicon.ico', mimetype='image/vnd.microsoft.icon'
+    )
+
+
+@app.route("/")
+@app.route("/home.html")
+@app.route("/index.html")
+def home():
+    return render_view('home.html.jinja2', {
+        'targets': config['targets'],
+        'planets': [s for s in config['targets'] if s['type'] == 'planet'],
+        'probes':  [s for s in config['targets'] if s['type'] == 'probe'],
+        'comets':  [s for s in config['targets'] if s['type'] == 'comet'],
+        'visits':  increment_hit_counter(),
+    })
+
+
+@app.route("/<source>_<started_at>_<stopped_at>.csv")
+def get_target_csv(source, started_at, stopped_at):
+    """
+    Grab data and orbit data for the specified `target`,
+    rearrange it and return it as a CSV file.
+    `started_at` and `stopped_at` should be UTC.
+    """
+    # http://cdpp1.cesr.fr/BASE/DDService/getDataUrl.php?dataSet=tao_ros_sw&StartTime=2014-02-23T10:00&StopTime=2016-02-24T23:59
+    # Process input parameters
+    source_config = get_source_config(source)
+    date_fmt = "%Y-%m-%dT%H:%M:%S"
+    try:
+        started_at = datetime.datetime.strptime(started_at, date_fmt)
+    except:
+        abort(400, "Invalid started_at parameter : '%s'." % started_at)
+    try:
+        stopped_at = datetime.datetime.strptime(stopped_at, date_fmt)
+    except:
+        abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at)
+
+    # todo: iterate on models when there are many
+    try:
+        model_slug = source_config['models'][0]['slug']
+    except:
+        abort(500, "Invalid model configuration for '%s'." % source)
+
+    filename = "%s_%s_%s.csv" % (source,
+                                 started_at.strftime(date_fmt),
+                                 stopped_at.strftime(date_fmt))
+
+    local_csv_file = get_path("../cache/%s" % filename)
+    if not isfile(local_csv_file):
+        with open(local_csv_file, mode="w+") as f:
+            f.write(generate_csv_contents(source_config,
+                                          started_at=started_at,
+                                          stopped_at=stopped_at))
+
+    if not isfile(local_csv_file):
+        abort(500, "Could not cache CSV file at '%s'." % local_csv_file)
+
+    return send_from_directory(get_path("../cache/"), filename)
+
+
+# @app.route("/<source>/data.csv")
+# def get_orbiter_csv(source):
+#     """
+#     DEPRECATED
+#     Grab data and orbit data for the specified `source`,
+#     rearrange it and return it as a CSV file.
+#     """
+#     # http://cdpp1.cesr.fr/BASE/DDService/getDataUrl.php?dataSet=tao_ros_sw&StartTime=2014-02-23T10:00&StopTime=2016-02-24T23:59
+#     # Process input parameters
+#     source_config = get_source_config(source)
+#     date_fmt = "%Y-%m-%dT%H:%M:%S"
+#     started_at = request.args.get('started_at')
+#     try:
+#         started_at = datetime.datetime.strptime(started_at, date_fmt)
+#     except:
+#         abort(400, "Invalid started_at parameter : '%s'." % started_at)
+#     stopped_at = request.args.get('stopped_at')
+#     try:
+#         stopped_at = datetime.datetime.strptime(stopped_at, date_fmt)
+#     except:
+#         abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at)
+#
+#     # todo: iterate on models when there are many
+#     try:
+#         model_slug = source_config['models'][0]['slug']
+#     except:
+#         abort(500, "Invalid model configuration for '%s'." % source)
+#
+#     # Grab the list of netCDF files from Myriam's API
+#     # http://cdpp.irap.omp.eu/BASE/DDService/getDataUrl.php?dataSet=jupiter_orb_all&StartTime=2014-02-23T10:00:10&StopTime=2017-02-24T23:59:00
+#     # http://cdpp.irap.omp.eu/BASE/DATA/TAO/JUPITER/SW/sw_2014.nc.gz
+#     model_files = retrieve_data(source, model_slug, started_at, stopped_at)
+#     orbits_files = retrieve_data(source, source_config['orbit']['model'], started_at, stopped_at)
+#
+#     si = StringIO.StringIO()
+#     cw = csv_writer(si)
+#     cw.writerow((  # the order matters !
+#         'time',
+#         'vrad', 'vtan', 'vlen',
+#         'magn', 'temp', 'pdyn', 'dens', 'angl',
+#         'xhci', 'yhci'
+#     ))
+#
+#     precision = "%Y-%m-%dT%H"  # model and orbits times are equal-ish
+#     orbits_data = {}  # keys are datetime as str, values arrays of XY
+#     for orbits_file in orbits_files:
+#         cdf_handle = Dataset(orbits_file, "r", format="NETCDF4")
+#         times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
+#         data_hci = cdf_handle.variables['HCI']
+#         for time, datum_hci in zip(times, data_hci):
+#             dtime = datetime_from_list(time)
+#             if started_at <= dtime <= stopped_at:
+#                 dkey = dtime.strftime(precision)
+#                 orbits_data[dkey] = datum_hci
+#     all_data = {}  # keys are datetime as str, values tuples of data
+#     for model_file in model_files:
+#         # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn
+#         cdf_handle = Dataset(model_file, "r", format="NETCDF4")
+#         times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
+#         data_v = cdf_handle.variables['V']
+#         data_b = cdf_handle.variables['B']
+#         data_t = cdf_handle.variables['T']
+#         data_n = cdf_handle.variables['N']
+#         data_p = cdf_handle.variables['P_dyn']
+#         data_d = cdf_handle.variables['Delta_angle']
+#         for time, datum_v, datum_b, datum_t, datum_p, datum_n, datum_d \
+#                 in zip(times, data_v, data_b, data_t, data_n, data_p, data_d):
+#             vrad = datum_v[0]
+#             vtan = datum_v[1]
+#             dtime = datetime_from_list(time)
+#             if started_at <= dtime <= stopped_at:
+#                 dkey = dtime.strftime(precision)
+#                 x_hci = None
+#                 y_hci = None
+#                 if dkey in orbits_data:
+#                     x_hci = orbits_data[dkey][0]
+#                     y_hci = orbits_data[dkey][1]
+#                 all_data[dkey] = (
+#                     dtime.strftime("%Y-%m-%dT%H:%M:%S+00:00"),
+#                     vrad, vtan, sqrt(vrad * vrad + vtan * vtan),
+#                     datum_b, datum_t, datum_n, datum_p, datum_d,
+#                     x_hci, y_hci
+#                 )
+#         cdf_handle.close()
+#
+#     for dkey in sorted(all_data):
+#         cw.writerow(all_data[dkey])
+#
+#     return si.getvalue()
+
+
 # DEV TOOLS ###################################################################
  
 @app.route("/inspect")
@@ -24,10 +24,12 @@
         return this$.parameters[p['id']] = p;
       });
     }
-    SpaceWeather.prototype.buildDataUrlForSource = function(source_slug){
+    SpaceWeather.prototype.buildDataUrlForSource = function(source_slug, started_at, stopped_at){
       var url;
       url = this.configuration['api']['data_for_interval'];
       url = url.replace('<source>', source_slug);
+      url = url.replace('<started_at>', started_at);
+      url = url.replace('<stopped_at>', stopped_at);
       return url;
     };
     SpaceWeather.prototype.addSource = function(source){
@@ -71,7 +73,7 @@
       });
     };
     SpaceWeather.prototype.init = function(){
-      var active_sources, res$, k, this$ = this;
+      var active_sources, res$, k, started_at, stopped_at, this$ = this;
       res$ = [];
       for (k in this.sources) {
         if (this.sources[k].config.active) {
@@ -80,8 +82,12 @@
       }
       active_sources = res$;
       this.orbits = new Orbits(this.configuration.orbits_container, this.configuration);
+      started_at = moment().subtract(1, 'years').hours(0).minutes(0).seconds(0);
+      stopped_at = moment().add(7, 'days').hours(0).minutes(0).seconds(0);
+      started_at = started_at.format("YYYY-MM-DDTHH:mm:ss");
+      stopped_at = stopped_at.format("YYYY-MM-DDTHH:mm:ss");
       active_sources.forEach(function(source){
-        return this$.loadData(source.slug, '2016-01-01T00:00:00', '2023-01-01T00:00:00').then(function(data){
+        return this$.loadData(source.slug, started_at, stopped_at).then(function(data){
           console.info("Loaded CSV data for " + source.slug + ".");
           this$.createTimeSeries(source, data);
           return this$.orbits.initOrbiter(source.slug, source.config, data['hci']);
@@ -98,8 +104,8 @@
       sw = this;
       promise = new Promise(function(resolve, reject){
         var url;
-        url = sw.buildDataUrlForSource(source_slug);
-        return d3.csv(url + ("?started_at=" + started_at + "&stopped_at=" + stopped_at), function(csv){
+        url = sw.buildDataUrlForSource(source_slug, started_at, stopped_at);
+        return d3.csv(url, function(csv){
           var timeFormat, data;
           timeFormat = d3.timeParse('%Y-%m-%dT%H:%M:%S%Z');
           data = {
@@ -32,9 +32,11 @@ export class SpaceWeather
         @parameters[p['id']] = p
     )
  
-  buildDataUrlForSource: (source_slug) ->
+  buildDataUrlForSource: (source_slug, started_at, stopped_at) ->
     url = @configuration['api']['data_for_interval']
     url = url.replace('<source>', source_slug)
+    url = url.replace('<started_at>', started_at)
+    url = url.replace('<stopped_at>', stopped_at)
     url
  
   addSource: (source) ->
@@ -63,8 +65,20 @@ export class SpaceWeather
   init: ->
     active_sources = [ @sources[k] for k of @sources when @sources[k].config.active ]
     @orbits = new Orbits(@configuration.orbits_container, @configuration)
+#    day = 864e5  # milliseconds
+#    now = new Date()
+#    started_at = new Date(+now - 365 * days)
+#    stopped_at = new Date(+now + 7 * days)
+#    year = new Date().getFullYear()
+#    started_at = "#{year - 1}-01-01T00:00:00"
+#    stopped_at = "#{year + 1}-01-01T00:00:00"
+    # Set the hours to zero so that files are cached for a day at most
+    started_at = moment().subtract(1, 'years').hours(0).minutes(0).seconds(0)
+    stopped_at = moment().add(7, 'days').hours(0).minutes(0).seconds(0)
+    started_at = started_at.format("YYYY-MM-DDTHH:mm:ss")
+    stopped_at = stopped_at.format("YYYY-MM-DDTHH:mm:ss")
     active_sources.forEach((source) ~>
-      @loadData(source.slug, '2016-01-01T00:00:00', '2023-01-01T00:00:00').then(
+      @loadData(source.slug, started_at, stopped_at).then(
         (data) ~>
           console.info "Loaded CSV data for #{source.slug}."
           @createTimeSeries(source, data)
@@ -78,8 +92,8 @@ export class SpaceWeather
   loadData: (source_slug, started_at, stopped_at) ->
     sw = this
     promise = new Promise((resolve, reject) ->
-      url = sw.buildDataUrlForSource(source_slug)
-      d3.csv(url+"?started_at=#{started_at}&stopped_at=#{stopped_at}", (csv) ->
+      url = sw.buildDataUrlForSource(source_slug, started_at, stopped_at)
+      d3.csv(url, (csv) ->
         timeFormat = d3.timeParse('%Y-%m-%dT%H:%M:%S%Z')
         data = {'hci': []};
         configuration['parameters'].forEach((parameter) ->
@@ -53,6 +53,16 @@
       <a class="mdl-navigation__link parameter" data-ts-slug="angl" href="#">Angle Source-Sun-Earth</a>
     </nav>
  
+    <div class="mdl-layout-spacer"></div>
+
+    <main class="mdl-layout__content">
+      <div class="page-content" style="text-align:center;font-style:italic;">
+        <p>
+          {{ visits }} visits since 2017
+        </p>
+      </div>
+    </main>
+
   </div>
   <main id="plots_wrapper" class="mdl-layout__content">
  
@@ -158,6 +168,7 @@
  
 {% block scripts_footer %}
 <script type="application/javascript" src="{{ static('js/vendor/d3.min.js') }}"></script>
+<script type="application/javascript" src="{{ static('js/vendor/moment.min.js') }}"></script>
 <script type="application/javascript" src="{{ static('js/swapp.js') }}"></script>
 <script type="application/javascript">
  
@@ -165,8 +176,7 @@ var configuration = {
   time_series_container: '#time_series',
   orbits_container: '#orbits',
   api : {
-      'data_for_interval': "{{ request.url_root }}<source>/data.csv"
-                           // ?started_at=<started_at>&stopped_at=<stopped_at>
+      'data_for_interval': "{{ request.url_root }}<source>_<started_at>_<stopped_at>.csv"
   },
   sun: {
     img: '{{ static('img/sun_128.png') }}'