Blame view

web/run.py 11.4 KB
9390ec89   Goutte   Initial experimen...
1
2
3
import random
import datetime
import StringIO
c42fea3a   Goutte   Rework the CSS wi...
4
from math import sqrt
9390ec89   Goutte   Initial experimen...
5
6
7
8
9

from os import listdir, environ
from os.path import isfile, join, abspath, dirname

import csv
8644387c   Goutte   Use real data.
10
11
12
import json
import gzip
import urllib
9390ec89   Goutte   Initial experimen...
13
14
15
16
from pprint import pprint
from csv import writer as csv_writer
from yaml import load as yaml_load
from flask import Flask
61179cdc   Goutte   Initial work on t...
17
from flask import redirect, url_for, send_from_directory, abort
9390ec89   Goutte   Initial experimen...
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
from flask import request
from jinja2 import Environment, FileSystemLoader
from netCDF4 import Dataset

# from model.Config import Config
# from model.News import NewsCollection, News


# PATH RELATIVITY #############################################################

THIS_DIRECTORY = dirname(abspath(__file__))


def get_path(relative_path):
    return abspath(join(THIS_DIRECTORY, relative_path))


# COLLECT GLOBAL INFORMATION FROM SOURCES #####################################

# VERSION
with open(get_path('../VERSION'), 'r') as version_file:
    version = version_file.read().strip()

# CONFIG
with open(get_path('../config.yml'), 'r') as config_file:
    config = yaml_load(config_file.read())


# SETUP FLASK ENGINE ##########################################################

app = Flask(__name__, root_path=THIS_DIRECTORY)
app.debug = environ.get('DEBUG') == 'true'


# SETUP JINJA2 TEMPLATE ENGINE ################################################

def static_global(filename):
    return url_for('static', filename=filename)


def shuffle_filter(seq):
    """
    This shuffles the sequence it is applied to.
    'tis a failure of jinja2 to not provide a shuffle filter by default.
    """
    try:
        result = list(seq)
        random.shuffle(result)
        return result
    except:
        return seq


def markdown_filter(value, nl2br=False, p=True):
    """
    nl2br: set to True to replace line breaks with <br> tags
    p: set to False to remove the enclosing <p></p> tags
    """
    from markdown import markdown
    from markdown.extensions.nl2br import Nl2BrExtension
    from markdown.extensions.abbr import AbbrExtension
    extensions = [AbbrExtension()]
    if nl2br is True:
        extensions.append(Nl2BrExtension())
    markdowned = markdown(value, output_format='html5', extensions=extensions)
    if p is False:
        markdowned = markdowned.replace(r"<p>", "").replace(r"</p>", "")
    return markdowned


tpl_engine = Environment(loader=FileSystemLoader([get_path('view')]),
                         trim_blocks=True,
                         lstrip_blocks=True)

tpl_engine.globals.update(
    url_for=url_for,
    static=static_global,
)

tpl_engine.filters['markdown'] = markdown_filter
tpl_engine.filters['md'] = markdown_filter
tpl_engine.filters['shuffle'] = shuffle_filter

tpl_global_vars = {
    'request': request,
    'version': version,
    'config': config,
    'now': datetime.datetime.now(),
}


# HELPERS #####################################################################

def render_view(view, context=None):
    """
    A simple helper to render [view] template with [context] vars.
    It automatically adds the global template vars defined above, too.
    It returns a string, usually the HTML contents to display.
    """
    context = {} if context is None else context
    return tpl_engine.get_template(view).render(
        dict(tpl_global_vars.items() + context.items())
    )


# def render_page(page, title="My Page", context=None):
#     """
#     A simple helper to render the md_page.html template with [context] vars &
#     the additional contents of `page/[page].md` in the `md_page` variable.
#     It automagically adds the global template vars defined above, too.
#     It returns a string, usually the HTML contents to display.
#     """
#     if context is None:
#         context = {}
#     context['title'] = title
#     context['md_page'] = ''
#     with file(get_path('page/%s.md' % page)) as f:
#         context['md_page'] = f.read()
#     return tpl_engine.get_template('md_page.html').render(
#         dict(tpl_global_vars.items() + context.items())
#     )

2d2af24b   Goutte   Add a basic orbit...
140
141
142
143
144
145
146
def datetime_from_list(time_list):
    # Day Of Year starts at 0, but for our datetime parser it starts at 1
    doy = '{:03d}'.format(int(''.join(time_list[4:7])) + 1)
    return datetime.datetime.strptime(
        "%s%s%s" % (''.join(time_list[0:4]), doy, ''.join(time_list[7:])),
        "%Y%j%H%M%S%f"
    )
9390ec89   Goutte   Initial experimen...
147

ce8af118   Goutte   Fix the favicon.
148

8644387c   Goutte   Use real data.
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
def get_source_config(slug):
    for s in config['sources']:
        if s['slug'] == slug:
            return s
    raise Exception("No source found for slug '%s'." % slug)


def retrieve_data(orbiter, what, started_at, stopped_at):
    """
    Handles remote querying Myriam's API, downloading, extracting and caching
    the netCDF files.
    :param orbiter: key of the source in the YAML config
    :param what: either 'model' or 'orbit', a key in the config of the source
    :param started_at:
    :param stopped_at:
    :return: a list of local file paths to netCDF (.nc) files
    """

    url = config['amda'].format(
        dataSet=what,
        startTime=started_at.isoformat(),
        stopTime=stopped_at.isoformat()
    )
    response = urllib.urlopen(url)
    remote_gzip_files = json.loads(response.read())
    if not remote_gzip_files or remote_gzip_files == 'NODATASET':
        abort(400, "Failed to fetch data at '%s'." % url)

    # retriever = urllib.URLopener()  # would we need to do this every time ?
    local_gzip_files = []
    for remote_gzip_file in remote_gzip_files:
        # removeme @Myriam
        if remote_gzip_file.endswith('/.gz'):
            continue
        remote_gzip_file = remote_gzip_file.replace('cdpp1', 'cdpp', 1)
        #########################
        filename = "%s_%s" % (orbiter, str(remote_gzip_file).split('/')[-1])
        local_gzip_file = get_path("../cache/%s" % filename)
        local_gzip_files.append(local_gzip_file)
        if not isfile(local_gzip_file):
            urllib.urlretrieve(remote_gzip_file, local_gzip_file)

    local_netc_files = []
    for local_gzip_file in local_gzip_files:
        local_netc_file = local_gzip_file[0:-3]
        local_netc_files.append(local_netc_file)
        with gzip.open(local_gzip_file, 'rb') as f:
            file_content = f.read()
            with open(local_netc_file, 'w+b') as g:
                g.write(file_content)

    return local_netc_files


9390ec89   Goutte   Initial experimen...
203
204
# ROUTING #####################################################################

ce8af118   Goutte   Fix the favicon.
205
206
207
208
209
210
211
212
@app.route('/favicon.ico')
def favicon():
    return send_from_directory(
        join(app.root_path, 'static', 'img'),
        'favicon.ico', mimetype='image/vnd.microsoft.icon'
    )


9390ec89   Goutte   Initial experimen...
213
214
215
216
@app.route("/")
@app.route("/home.html")
@app.route("/index.html")
def home():
a79c5268   Goutte   Add probes and co...
217
218
219
220
221
222
    return render_view('home.html.jinja2', {
        'sources': config['sources'],
        'planets': [s for s in config['sources'] if s['type'] == 'planet'],
        'probes':  [s for s in config['sources'] if s['type'] == 'probe'],
        'comets':  [s for s in config['sources'] if s['type'] == 'comet'],
    })
9390ec89   Goutte   Initial experimen...
223
224
225
226
227


@app.route("/inspect")
def analyze_cdf():
    cdf_to_inspect = get_path("../res/dummy.nc")
2d2af24b   Goutte   Add a basic orbit...
228
    cdf_to_inspect = get_path("../res/dummy_jupiter_coordinates.nc")
9390ec89   Goutte   Initial experimen...
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247

    si = StringIO.StringIO()
    cw = csv.DictWriter(si, fieldnames=['Name', 'Shape', 'Length'])
    cw.writeheader()

    # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn, QualityFlag
    cdf_handle = Dataset(cdf_to_inspect, "r", format="NETCDF4")
    for variable in cdf_handle.variables:
        v = cdf_handle.variables[variable]
        cw.writerow({
            'Name': variable,
            'Shape': v.shape,
            'Length': v.size,
        })
    cdf_handle.close()

    return si.getvalue()


61179cdc   Goutte   Initial work on t...
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
@app.route("/<orbiter>_data.csv")
def get_orbiter_csv(orbiter):
    # http://cdpp1.cesr.fr/BASE/DDService/getDataUrl.php?dataSet=tao_ros_sw&StartTime=2014-02-23T10:00&StopTime=2016-02-24T23:59
    # Process input parameters
    if orbiter not in config['models']:
        abort(400, "Invalid orbiter '%s'." % orbiter)
    date_fmt = "%Y-%m-%dT%H:%M:%S"
    started_at = request.args.get('started_at')
    try:
        started_at = datetime.datetime.strptime(started_at, date_fmt)
    except:
        abort(400, "Invalid started_at parameter : '%s'." % started_at)
    stopped_at = request.args.get('stopped_at')
    try:
        stopped_at = datetime.datetime.strptime(stopped_at, date_fmt)
    except:
        abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at)

    # Grab the list of netCDF files from Myriam's API todo
a79c5268   Goutte   Add probes and co...
267
    #
8644387c   Goutte   Use real data.
268
269
270
271
272
273
274
275
276
277
    # http://cdpp.irap.omp.eu/BASE/DDService/getDataUrl.php?dataSet=jupiter_orb_all&StartTime=2014-02-23T10:00:10&StopTime=2017-02-24T23:59:00
    # http://cdpp.irap.omp.eu/BASE/DATA/TAO/JUPITER/SW/sw_2014.nc.gz

    source_config = get_source_config(orbiter)
    try:
        model_slug = source_config['models'][0]['slug']
    except:
        abort(500, "Invalid model configuration for '%s'." % orbiter)
    model_files = retrieve_data(orbiter, model_slug, started_at, stopped_at)
    orbits_files = retrieve_data(orbiter, source_config['orbit'], started_at, stopped_at)
61179cdc   Goutte   Initial work on t...
278
279
280

    si = StringIO.StringIO()
    cw = csv_writer(si)
8644387c   Goutte   Use real data.
281
    cw.writerow((  # the order matters !
61179cdc   Goutte   Initial work on t...
282
283
        'time',
        'vrad', 'vtan', 'vlen',
8644387c   Goutte   Use real data.
284
285
        'magn', 'temp', 'pdyn', 'dens', 'angl',
        'xhci', 'yhci'
61179cdc   Goutte   Initial work on t...
286
    ))
9390ec89   Goutte   Initial experimen...
287

8644387c   Goutte   Use real data.
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
    precision = "%Y-%m-%dT%H"  # model and orbits times are equal-ish
    orbits_data = {}  # keys are datetime as str, values arrays of XY
    for orbits_file in orbits_files:
        cdf_handle = Dataset(orbits_file, "r", format="NETCDF4")
        times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
        data_hci = cdf_handle.variables['HCI']
        for time, datum_hci in zip(times, data_hci):
            dtime = datetime_from_list(time)
            if started_at <= dtime <= stopped_at:
                dkey = dtime.strftime(precision)
                orbits_data[dkey] = datum_hci
    all_data = {}  # keys are datetime as str, values tuples of data
    for model_file in model_files:
        # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn
        cdf_handle = Dataset(model_file, "r", format="NETCDF4")
        times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
        data_v = cdf_handle.variables['V']
        data_b = cdf_handle.variables['B']
        data_t = cdf_handle.variables['T']
        data_n = cdf_handle.variables['N']
        data_p = cdf_handle.variables['P_dyn']
        data_d = cdf_handle.variables['Delta_angle']
        for time, datum_v, datum_b, datum_t, datum_p, datum_n, datum_d \
                in zip(times, data_v, data_b, data_t, data_n, data_p, data_d):
            vrad = datum_v[0]
            vtan = datum_v[1]
            dtime = datetime_from_list(time)
            if started_at <= dtime <= stopped_at:
                dkey = dtime.strftime(precision)
                x_hci = None
                y_hci = None
                if dkey in orbits_data:
                    x_hci = orbits_data[dkey][0]
                    y_hci = orbits_data[dkey][1]
                all_data[dkey] = (
                    dtime.strftime("%Y-%m-%dT%H:%M:%S+00:00"),
                    vrad, vtan, sqrt(vrad * vrad + vtan * vtan),
                    datum_b, datum_t, datum_n, datum_p, datum_d,
                    x_hci, y_hci
                )
        cdf_handle.close()

    for dkey in sorted(all_data):
        cw.writerow(all_data[dkey])
2d2af24b   Goutte   Add a basic orbit...
332
333
334

    return si.getvalue()

8644387c   Goutte   Use real data.
335

9390ec89   Goutte   Initial experimen...
336
337
338
339
340
341
# MAIN ########################################################################

if __name__ == "__main__":
    # Debug mode on, as the production server does not use this.
    extra_files = [get_path('../config.yml')]
    app.run(debug=True, extra_files=extra_files)