Use real data.

Goutte
1 parent c15a50cb
Showing 4 changed files with 208 additions and 90 deletions Show diff stats
CHANGELOG.md
config.yml
web/run.py
web/view/home.html.jinja2
+- Start/Stop datetime fields
+- Play button to start time dimension
+- Download raw data for time interval
+- Same via SAMP
+
+- Planets
+- Probes
+    - Rosetta
+    - Juno
+- Comets (the one visited by rosetta)
+
 ## 0.0.0
  
 - Initial website skeleton
-- Initial data bridge between netCDF and CSV
 \ No newline at end of file
+- Initial data bridge between netCDF and CSV
+- New parameters
+    - Temperature `T` (K)
+    - Density `N` (cm^-3)
+    - Angle Planet-Sun-Earth `Delta_angle` (degrees)
+    - Magnetism => B Tangential
+- Orbits axes titles
+- Logo Europlanet
@@ -35,6 +35,27 @@ authors:
       role: Project Coordinator
       mail: vincent.genot@irap.omp.eu
  
+
+amda: "http://cdpp.irap.omp.eu/BASE/DDService/getDataUrl.php?dataSet={dataSet}&StartTime={startTime}&StopTime={stopTime}"
+
+#Targets
+#Earth 		earth_orb_all
+#Jupiter 	jupiter_orb_all
+#Mars		mars_orb_all
+#Mercury		mercury_orb_all
+#Neptune		neptune_orb_all
+#Venus		venus_orb_all
+#Uranus		uranus_orb_all
+#
+#Comets
+#67P		p67_orb_all
+#
+#Probes
+#Maven 		maven_orb_cruise
+#Juno		juno_cruise_all
+#Exomars		exomars_cruise_all
+#Rosetta		ros_orb_cruise
+
 # `slug` used internally, and should match [a-z0-9]+
 # `name` is displayed
 # `title` appears on mouse hover
@@ -45,54 +66,94 @@ sources:
       slug: 'mercury'
       name: 'Mercury'
       title: 'Mercury (coming soon)'
+      orbit: 'mercury_orb_all'
+      models:
+          - slug: 'tao_mercury_sw'
+            started_at: "1990-01-01T01:30:00"
+            stopped_at: "2017-02-19T00:00:00"
       locked: true
       default: true
     - type: 'planet'
       slug: 'venus'
       name: 'Venus'
       title: 'Venus (coming soon)'
+      orbit: 'venus_orb_all'
+      models:
+          - slug: 'tao_venus_sw'
+            started_at: "1990-01-01T01:30:00"
+            stopped_at: "2017-02-19T00:00:00"
       locked: true
       default: true
     - type: 'planet'
       slug: 'earth'
       name: 'Earth'
       title: 'Earth (coming soon)'
+      orbit: 'earth_orb_all'
+      #model: 'tao_earth_sw'
       locked: true
       default: true
     - type: 'planet'
       slug: 'mars'
       name: 'Mars'
       title: 'Mars (coming soon)'
+      orbit: 'mars_orb_all'
+      models:
+          - slug: 'tao_mars_sw'
+            started_at: "1990-01-01T01:30:00"
+            stopped_at: "2017-02-19T00:00:00"
       locked: true
       default: true
     - type: 'planet'
       slug: 'jupiter'
       name: 'Jupiter'
       title: 'Jupiter'
+      orbit: 'jupiter_orb_all'
+      models:
+          - slug: 'tao_jup_sw'
+            started_at: "1990-01-01T01:30:00"
+            stopped_at: "2017-02-19T00:00:00"
+      started_at: "1990-01-01T01:30:00"
+      stopped_at: "2017-02-19T00:00:00"
       locked: false
       default: true
     - type: 'planet'
       slug: 'saturn'
       name: 'Saturn'
       title: 'Saturn (coming soon)'
+      orbit: 'saturn_orb_all'
+      models:
+          - slug: 'tao_sat_sw'
+            started_at: "1990-01-01T01:30:00"
+            stopped_at: "2017-02-19T00:00:00"
       locked: true
       default: true
     - type: 'probe'
       slug: 'rosetta'
       name: 'Rosetta'
       title: 'Rosetta (coming soon)'
+      orbit: 'ros_orb_cruise'
+      models:
+          - slug: 'tao_ros_sw'
+            started_at: "1990-01-01T01:30:00"
+            stopped_at: "2017-02-19T00:00:00"
       locked: true
       default: false
     - type: 'probe'
       slug: 'juno'
       name: 'Juno'
       title: 'Juno (coming soon)'
+      orbit: 'juno_cruise_all'
+      models:
+          - slug: 'tao_juno_sw'
+            started_at: "1990-01-01T01:30:00"
+            stopped_at: "2017-02-19T00:00:00"
       locked: true
       default: false
     - type: 'comet'
       slug: 'tchouri'
       name: 'Tchouri'
       title: 'Tchouri (coming soon)'
+      orbit: 'p67_orb_all'
       locked: true
       default: false
  
@@ -7,6 +7,9 @@ from os import listdir, environ
 from os.path import isfile, join, abspath, dirname
  
 import csv
+import json
+import gzip
+import urllib
 from pprint import pprint
 from csv import writer as csv_writer
 from yaml import load as yaml_load
@@ -143,6 +146,60 @@ def datetime_from_list(time_list):
     )
  
  
+def get_source_config(slug):
+    for s in config['sources']:
+        if s['slug'] == slug:
+            return s
+    raise Exception("No source found for slug '%s'." % slug)
+
+
+def retrieve_data(orbiter, what, started_at, stopped_at):
+    """
+    Handles remote querying Myriam's API, downloading, extracting and caching
+    the netCDF files.
+    :param orbiter: key of the source in the YAML config
+    :param what: either 'model' or 'orbit', a key in the config of the source
+    :param started_at:
+    :param stopped_at:
+    :return: a list of local file paths to netCDF (.nc) files
+    """
+
+    url = config['amda'].format(
+        dataSet=what,
+        startTime=started_at.isoformat(),
+        stopTime=stopped_at.isoformat()
+    )
+    response = urllib.urlopen(url)
+    remote_gzip_files = json.loads(response.read())
+    if not remote_gzip_files or remote_gzip_files == 'NODATASET':
+        abort(400, "Failed to fetch data at '%s'." % url)
+
+    # retriever = urllib.URLopener()  # would we need to do this every time ?
+    local_gzip_files = []
+    for remote_gzip_file in remote_gzip_files:
+        # removeme @Myriam
+        if remote_gzip_file.endswith('/.gz'):
+            continue
+        remote_gzip_file = remote_gzip_file.replace('cdpp1', 'cdpp', 1)
+        #########################
+        filename = "%s_%s" % (orbiter, str(remote_gzip_file).split('/')[-1])
+        local_gzip_file = get_path("../cache/%s" % filename)
+        local_gzip_files.append(local_gzip_file)
+        if not isfile(local_gzip_file):
+            urllib.urlretrieve(remote_gzip_file, local_gzip_file)
+
+    local_netc_files = []
+    for local_gzip_file in local_gzip_files:
+        local_netc_file = local_gzip_file[0:-3]
+        local_netc_files.append(local_netc_file)
+        with gzip.open(local_gzip_file, 'rb') as f:
+            file_content = f.read()
+            with open(local_netc_file, 'w+b') as g:
+                g.write(file_content)
+
+    return local_netc_files
+
+
 # ROUTING #####################################################################
  
 @app.route('/favicon.ico')
@@ -188,40 +245,6 @@ def analyze_cdf():
     return si.getvalue()
  
  
-@app.route("/test.csv")
-def get_csv():
-    si = StringIO.StringIO()
-    cw = csv_writer(si)
-
-    # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn, QualityFlag
-    cdf_handle = Dataset(get_path("../res/dummy.nc"), "r", format="NETCDF4")
-    # YYYY DOY HH MM SS .ms
-    times = cdf_handle.variables['Time']
-    data_v = cdf_handle.variables['V']
-    data_b = cdf_handle.variables['B']
-    data_t = cdf_handle.variables['T']
-    data_n = cdf_handle.variables['N']
-    data_p = cdf_handle.variables['P_dyn']
-    data_a = cdf_handle.variables['Delta_angle']
-    cw.writerow((
-        'time',
-        'vrad', 'vtan', 'vlen',
-        'magn', 'temp', 'pdyn', 'dens', 'angl'
-    ))
-    for time, datum_v, datum_b, datum_t, datum_p, datum_n, datum_a in \
-            zip(times, data_v, data_b, data_t, data_n, data_p, data_a):
-        vrad = datum_v[0]
-        vtan = datum_v[1]
-        cw.writerow((
-            datetime_from_list(time).strftime("%Y-%m-%dT%H:%M:%S+00:00"),
-            vrad, vtan, sqrt(vrad * vrad + vtan * vtan),
-            datum_b, datum_t, datum_n, datum_p, datum_a
-        ))
-    cdf_handle.close()
-
-    return si.getvalue()
-
-
 @app.route("/<orbiter>_data.csv")
 def get_orbiter_csv(orbiter):
     # http://cdpp1.cesr.fr/BASE/DDService/getDataUrl.php?dataSet=tao_ros_sw&StartTime=2014-02-23T10:00&StopTime=2016-02-24T23:59
@@ -242,55 +265,74 @@ def get_orbiter_csv(orbiter):
  
     # Grab the list of netCDF files from Myriam's API todo
     #
+    # http://cdpp.irap.omp.eu/BASE/DDService/getDataUrl.php?dataSet=jupiter_orb_all&StartTime=2014-02-23T10:00:10&StopTime=2017-02-24T23:59:00
+    # http://cdpp.irap.omp.eu/BASE/DATA/TAO/JUPITER/SW/sw_2014.nc.gz
+
+    source_config = get_source_config(orbiter)
+    try:
+        model_slug = source_config['models'][0]['slug']
+    except:
+        abort(500, "Invalid model configuration for '%s'." % orbiter)
+    model_files = retrieve_data(orbiter, model_slug, started_at, stopped_at)
+    orbits_files = retrieve_data(orbiter, source_config['orbit'], started_at, stopped_at)
  
     si = StringIO.StringIO()
     cw = csv_writer(si)
-
-    # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn, QualityFlag
-    cdf_handle = Dataset(get_path("../res/dummy.nc"), "r", format="NETCDF4")
-
-    # YYYY DOY HH MM SS .ms
-    times = cdf_handle.variables['Time']
-    data_v = cdf_handle.variables['V']
-    data_b = cdf_handle.variables['B']
-    data_p = cdf_handle.variables['P_dyn']
-    cw.writerow((
+    cw.writerow((  # the order matters !
         'time',
         'vrad', 'vtan', 'vlen',
-        'magn', 'pdyn'
+        'magn', 'temp', 'pdyn', 'dens', 'angl',
+        'xhci', 'yhci'
     ))
-    for time, datum_v, datum_b, datum_p in zip(times, data_v, data_b, data_p):
-        vrad = datum_v[0]
-        vtan = datum_v[1]
-        cw.writerow((
-            datetime_from_list(time).strftime("%Y-%m-%dT%H:%M:%S+00:00"),
-            vrad, vtan, sqrt(vrad * vrad + vtan * vtan),
-            datum_b, datum_p
-        ))
-    cdf_handle.close()
  
-    return si.getvalue()
-
-
-@app.route("/astral_coordinates.csv")
-def get_astral_coordinates_csv():
-    si = StringIO.StringIO()
-    cw = csv_writer(si)
-
-    # Time, StartTime, StopTime, XYZ_HCI, XYZ_IAU_SUN, XYZ_HEE
-    cdf_handle = Dataset(get_path("../res/dummy_jupiter_coordinates.nc"), "r", format="NETCDF4")
-    times = cdf_handle.variables['Time']
-    data_xyz_hci = cdf_handle.variables['XYZ_HCI']
-    cw.writerow(('time', 'x_hci', 'y_hci'))
-    for time, datum_xyz_hci in zip(times, data_xyz_hci):
-        cw.writerow((
-            datetime_from_list(time).strftime("%Y-%m-%dT%H:%M:%S+00:00"),
-            datum_xyz_hci[0], datum_xyz_hci[1]
-        ))
-    cdf_handle.close()
+    precision = "%Y-%m-%dT%H"  # model and orbits times are equal-ish
+    orbits_data = {}  # keys are datetime as str, values arrays of XY
+    for orbits_file in orbits_files:
+        cdf_handle = Dataset(orbits_file, "r", format="NETCDF4")
+        times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
+        data_hci = cdf_handle.variables['HCI']
+        for time, datum_hci in zip(times, data_hci):
+            dtime = datetime_from_list(time)
+            if started_at <= dtime <= stopped_at:
+                dkey = dtime.strftime(precision)
+                orbits_data[dkey] = datum_hci
+    all_data = {}  # keys are datetime as str, values tuples of data
+    for model_file in model_files:
+        # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn
+        cdf_handle = Dataset(model_file, "r", format="NETCDF4")
+        times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
+        data_v = cdf_handle.variables['V']
+        data_b = cdf_handle.variables['B']
+        data_t = cdf_handle.variables['T']
+        data_n = cdf_handle.variables['N']
+        data_p = cdf_handle.variables['P_dyn']
+        data_d = cdf_handle.variables['Delta_angle']
+        for time, datum_v, datum_b, datum_t, datum_p, datum_n, datum_d \
+                in zip(times, data_v, data_b, data_t, data_n, data_p, data_d):
+            vrad = datum_v[0]
+            vtan = datum_v[1]
+            dtime = datetime_from_list(time)
+            if started_at <= dtime <= stopped_at:
+                dkey = dtime.strftime(precision)
+                x_hci = None
+                y_hci = None
+                if dkey in orbits_data:
+                    x_hci = orbits_data[dkey][0]
+                    y_hci = orbits_data[dkey][1]
+                all_data[dkey] = (
+                    dtime.strftime("%Y-%m-%dT%H:%M:%S+00:00"),
+                    vrad, vtan, sqrt(vrad * vrad + vtan * vtan),
+                    datum_b, datum_t, datum_n, datum_p, datum_d,
+                    x_hci, y_hci
+                )
+        cdf_handle.close()
+
+    for dkey in sorted(all_data):
+        cw.writerow(all_data[dkey])
  
     return si.getvalue()
  
+
 # MAIN ########################################################################
  
 if __name__ == "__main__":
@@ -176,16 +176,20 @@ jQuery().ready(function($){
  
   var sw = new SpaceWeather(configuration);
  
-  d3.csv('{{ url_for('get_csv') }}', function(csv){
+  d3.csv('{{ url_for('get_orbiter_csv', orbiter='jupiter', started_at='2016-03-19T00:00:00', stopped_at='2022-01-01T00:00:00') }}', function(csv){
     var timeFormat = d3.timeParse('%Y-%m-%dT%H:%M:%S%Z');
-    var data = {'pdyn': [], 'magn': [], 'vlen': [], 'temp': [], 'dens': [], 'angl': []};
+    var data = {'pdyn': [], 'magn': [], 'vlen': [], 'temp': [], 'dens': [], 'angl': [], 'hci': []};
     csv.forEach(function (d) {
-      data['pdyn'].push({x: timeFormat(d['time']), y: parseFloat(d['pdyn'])});
-      data['magn'].push({x: timeFormat(d['time']), y: parseFloat(d['magn'])});
-      data['vlen'].push({x: timeFormat(d['time']), y: parseFloat(d['vlen'])});
-      data['temp'].push({x: timeFormat(d['time']), y: parseFloat(d['temp'])});
-      data['dens'].push({x: timeFormat(d['time']), y: parseFloat(d['dens'])});
-      data['angl'].push({x: timeFormat(d['time']), y: parseFloat(d['angl'])});
+      var dtime = timeFormat(d['time']);
+      data['pdyn'].push({x: dtime, y: parseFloat(d['pdyn'])});
+      data['magn'].push({x: dtime, y: parseFloat(d['magn'])});
+      data['vlen'].push({x: dtime, y: parseFloat(d['vlen'])});
+      data['temp'].push({x: dtime, y: parseFloat(d['temp'])});
+      data['dens'].push({x: dtime, y: parseFloat(d['dens'])});
+      data['angl'].push({x: dtime, y: parseFloat(d['angl'])});
+      if (d['xhci'] && d['yhci']) {
+        data['hci'].push({t: dtime, x: parseFloat(d['xhci']), y: parseFloat(d['yhci'])});
+      }
     });
  
     var container = '#time_series';
@@ -221,16 +225,9 @@ jQuery().ready(function($){
       };
     });
  
-  });
  
-  d3.csv('{{ url_for('get_astral_coordinates_csv') }}', function(csv){
-    var timeFormat = d3.timeParse('%Y-%m-%dT%H:%M:%S%Z');
-    var data = [];
-    csv.forEach(function (d) {
-      data.push({t: timeFormat(d['time']), x: parseFloat(d['x_hci']), y: parseFloat(d['y_hci'])});
-    });
+    orbits = new Orbits(configuration.sources, data['hci'], configuration.orbits_container);
  
-    orbits = new Orbits(configuration.sources, data, '#orbits');
   });
  
   var toggleTimeSeries = function(slug) {