Blame view

web/run.py 11.6 KB
9390ec89   Goutte   Initial experimen...
1
2
3
import random
import datetime
import StringIO
c42fea3a   Goutte   Rework the CSS wi...
4
from math import sqrt
9390ec89   Goutte   Initial experimen...
5
6
7
8
9

from os import listdir, environ
from os.path import isfile, join, abspath, dirname

import csv
8644387c   Goutte   Use real data.
10
11
12
import json
import gzip
import urllib
9390ec89   Goutte   Initial experimen...
13
14
15
16
from pprint import pprint
from csv import writer as csv_writer
from yaml import load as yaml_load
from flask import Flask
61179cdc   Goutte   Initial work on t...
17
from flask import redirect, url_for, send_from_directory, abort
9390ec89   Goutte   Initial experimen...
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
from flask import request
from jinja2 import Environment, FileSystemLoader
from netCDF4 import Dataset

# from model.Config import Config
# from model.News import NewsCollection, News


# PATH RELATIVITY #############################################################

THIS_DIRECTORY = dirname(abspath(__file__))


def get_path(relative_path):
    return abspath(join(THIS_DIRECTORY, relative_path))


# COLLECT GLOBAL INFORMATION FROM SOURCES #####################################

# VERSION
with open(get_path('../VERSION'), 'r') as version_file:
    version = version_file.read().strip()

# CONFIG
with open(get_path('../config.yml'), 'r') as config_file:
    config = yaml_load(config_file.read())


# SETUP FLASK ENGINE ##########################################################

app = Flask(__name__, root_path=THIS_DIRECTORY)
app.debug = environ.get('DEBUG') == 'true'


# SETUP JINJA2 TEMPLATE ENGINE ################################################

def static_global(filename):
    return url_for('static', filename=filename)


def shuffle_filter(seq):
    """
    This shuffles the sequence it is applied to.
    'tis a failure of jinja2 to not provide a shuffle filter by default.
    """
    try:
        result = list(seq)
        random.shuffle(result)
        return result
    except:
        return seq


def markdown_filter(value, nl2br=False, p=True):
    """
    nl2br: set to True to replace line breaks with <br> tags
    p: set to False to remove the enclosing <p></p> tags
    """
    from markdown import markdown
    from markdown.extensions.nl2br import Nl2BrExtension
    from markdown.extensions.abbr import AbbrExtension
    extensions = [AbbrExtension()]
    if nl2br is True:
        extensions.append(Nl2BrExtension())
    markdowned = markdown(value, output_format='html5', extensions=extensions)
    if p is False:
        markdowned = markdowned.replace(r"<p>", "").replace(r"</p>", "")
    return markdowned


tpl_engine = Environment(loader=FileSystemLoader([get_path('view')]),
                         trim_blocks=True,
                         lstrip_blocks=True)

tpl_engine.globals.update(
    url_for=url_for,
    static=static_global,
)

tpl_engine.filters['markdown'] = markdown_filter
tpl_engine.filters['md'] = markdown_filter
tpl_engine.filters['shuffle'] = shuffle_filter

tpl_global_vars = {
    'request': request,
    'version': version,
    'config': config,
    'now': datetime.datetime.now(),
}


# HELPERS #####################################################################

def render_view(view, context=None):
    """
    A simple helper to render [view] template with [context] vars.
    It automatically adds the global template vars defined above, too.
    It returns a string, usually the HTML contents to display.
    """
    context = {} if context is None else context
    return tpl_engine.get_template(view).render(
        dict(tpl_global_vars.items() + context.items())
    )


# def render_page(page, title="My Page", context=None):
#     """
#     A simple helper to render the md_page.html template with [context] vars &
#     the additional contents of `page/[page].md` in the `md_page` variable.
#     It automagically adds the global template vars defined above, too.
#     It returns a string, usually the HTML contents to display.
#     """
#     if context is None:
#         context = {}
#     context['title'] = title
#     context['md_page'] = ''
#     with file(get_path('page/%s.md' % page)) as f:
#         context['md_page'] = f.read()
#     return tpl_engine.get_template('md_page.html').render(
#         dict(tpl_global_vars.items() + context.items())
#     )

2d2af24b   Goutte   Add a basic orbit...
140
def datetime_from_list(time_list):
0b9821dd   Goutte   Clean up.
141
142
143
144
    """
    Datetimes in retrieved CDFs are stored in lists of numbers,
    with DayOfYear starting at 0. We want it starting at 1 for default parsers.
    """
2d2af24b   Goutte   Add a basic orbit...
145
146
147
148
149
150
    # Day Of Year starts at 0, but for our datetime parser it starts at 1
    doy = '{:03d}'.format(int(''.join(time_list[4:7])) + 1)
    return datetime.datetime.strptime(
        "%s%s%s" % (''.join(time_list[0:4]), doy, ''.join(time_list[7:])),
        "%Y%j%H%M%S%f"
    )
9390ec89   Goutte   Initial experimen...
151

ce8af118   Goutte   Fix the favicon.
152

8644387c   Goutte   Use real data.
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
def get_source_config(slug):
    for s in config['sources']:
        if s['slug'] == slug:
            return s
    raise Exception("No source found for slug '%s'." % slug)


def retrieve_data(orbiter, what, started_at, stopped_at):
    """
    Handles remote querying Myriam's API, downloading, extracting and caching
    the netCDF files.
    :param orbiter: key of the source in the YAML config
    :param what: either 'model' or 'orbit', a key in the config of the source
    :param started_at:
    :param stopped_at:
    :return: a list of local file paths to netCDF (.nc) files
    """

    url = config['amda'].format(
        dataSet=what,
        startTime=started_at.isoformat(),
        stopTime=stopped_at.isoformat()
    )
    response = urllib.urlopen(url)
    remote_gzip_files = json.loads(response.read())
    if not remote_gzip_files or remote_gzip_files == 'NODATASET':
        abort(400, "Failed to fetch data at '%s'." % url)

    # retriever = urllib.URLopener()  # would we need to do this every time ?
    local_gzip_files = []
    for remote_gzip_file in remote_gzip_files:
        # removeme @Myriam
        if remote_gzip_file.endswith('/.gz'):
            continue
        remote_gzip_file = remote_gzip_file.replace('cdpp1', 'cdpp', 1)
        #########################
        filename = "%s_%s" % (orbiter, str(remote_gzip_file).split('/')[-1])
        local_gzip_file = get_path("../cache/%s" % filename)
        local_gzip_files.append(local_gzip_file)
        if not isfile(local_gzip_file):
            urllib.urlretrieve(remote_gzip_file, local_gzip_file)

    local_netc_files = []
    for local_gzip_file in local_gzip_files:
        local_netc_file = local_gzip_file[0:-3]
        local_netc_files.append(local_netc_file)
        with gzip.open(local_gzip_file, 'rb') as f:
            file_content = f.read()
            with open(local_netc_file, 'w+b') as g:
                g.write(file_content)

    return local_netc_files


9390ec89   Goutte   Initial experimen...
207
208
# ROUTING #####################################################################

ce8af118   Goutte   Fix the favicon.
209
210
211
212
213
214
215
216
@app.route('/favicon.ico')
def favicon():
    return send_from_directory(
        join(app.root_path, 'static', 'img'),
        'favicon.ico', mimetype='image/vnd.microsoft.icon'
    )


9390ec89   Goutte   Initial experimen...
217
218
219
220
@app.route("/")
@app.route("/home.html")
@app.route("/index.html")
def home():
a79c5268   Goutte   Add probes and co...
221
222
223
224
225
226
    return render_view('home.html.jinja2', {
        'sources': config['sources'],
        'planets': [s for s in config['sources'] if s['type'] == 'planet'],
        'probes':  [s for s in config['sources'] if s['type'] == 'probe'],
        'comets':  [s for s in config['sources'] if s['type'] == 'comet'],
    })
9390ec89   Goutte   Initial experimen...
227
228
229
230
231


@app.route("/inspect")
def analyze_cdf():
    cdf_to_inspect = get_path("../res/dummy.nc")
2d2af24b   Goutte   Add a basic orbit...
232
    cdf_to_inspect = get_path("../res/dummy_jupiter_coordinates.nc")
9390ec89   Goutte   Initial experimen...
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251

    si = StringIO.StringIO()
    cw = csv.DictWriter(si, fieldnames=['Name', 'Shape', 'Length'])
    cw.writeheader()

    # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn, QualityFlag
    cdf_handle = Dataset(cdf_to_inspect, "r", format="NETCDF4")
    for variable in cdf_handle.variables:
        v = cdf_handle.variables[variable]
        cw.writerow({
            'Name': variable,
            'Shape': v.shape,
            'Length': v.size,
        })
    cdf_handle.close()

    return si.getvalue()


61179cdc   Goutte   Initial work on t...
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
@app.route("/<orbiter>_data.csv")
def get_orbiter_csv(orbiter):
    # http://cdpp1.cesr.fr/BASE/DDService/getDataUrl.php?dataSet=tao_ros_sw&StartTime=2014-02-23T10:00&StopTime=2016-02-24T23:59
    # Process input parameters
    if orbiter not in config['models']:
        abort(400, "Invalid orbiter '%s'." % orbiter)
    date_fmt = "%Y-%m-%dT%H:%M:%S"
    started_at = request.args.get('started_at')
    try:
        started_at = datetime.datetime.strptime(started_at, date_fmt)
    except:
        abort(400, "Invalid started_at parameter : '%s'." % started_at)
    stopped_at = request.args.get('stopped_at')
    try:
        stopped_at = datetime.datetime.strptime(stopped_at, date_fmt)
    except:
        abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at)

8644387c   Goutte   Use real data.
270
271

    source_config = get_source_config(orbiter)
28ef3790   Goutte   Clean up.
272
    # todo: iterate on models when there are many
8644387c   Goutte   Use real data.
273
274
275
276
    try:
        model_slug = source_config['models'][0]['slug']
    except:
        abort(500, "Invalid model configuration for '%s'." % orbiter)
28ef3790   Goutte   Clean up.
277
278
279
280

    # Grab the list of netCDF files from Myriam's API
    # http://cdpp.irap.omp.eu/BASE/DDService/getDataUrl.php?dataSet=jupiter_orb_all&StartTime=2014-02-23T10:00:10&StopTime=2017-02-24T23:59:00
    # http://cdpp.irap.omp.eu/BASE/DATA/TAO/JUPITER/SW/sw_2014.nc.gz
8644387c   Goutte   Use real data.
281
282
    model_files = retrieve_data(orbiter, model_slug, started_at, stopped_at)
    orbits_files = retrieve_data(orbiter, source_config['orbit'], started_at, stopped_at)
61179cdc   Goutte   Initial work on t...
283
284
285

    si = StringIO.StringIO()
    cw = csv_writer(si)
8644387c   Goutte   Use real data.
286
    cw.writerow((  # the order matters !
61179cdc   Goutte   Initial work on t...
287
288
        'time',
        'vrad', 'vtan', 'vlen',
8644387c   Goutte   Use real data.
289
290
        'magn', 'temp', 'pdyn', 'dens', 'angl',
        'xhci', 'yhci'
61179cdc   Goutte   Initial work on t...
291
    ))
9390ec89   Goutte   Initial experimen...
292

8644387c   Goutte   Use real data.
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
    precision = "%Y-%m-%dT%H"  # model and orbits times are equal-ish
    orbits_data = {}  # keys are datetime as str, values arrays of XY
    for orbits_file in orbits_files:
        cdf_handle = Dataset(orbits_file, "r", format="NETCDF4")
        times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
        data_hci = cdf_handle.variables['HCI']
        for time, datum_hci in zip(times, data_hci):
            dtime = datetime_from_list(time)
            if started_at <= dtime <= stopped_at:
                dkey = dtime.strftime(precision)
                orbits_data[dkey] = datum_hci
    all_data = {}  # keys are datetime as str, values tuples of data
    for model_file in model_files:
        # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn
        cdf_handle = Dataset(model_file, "r", format="NETCDF4")
        times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
        data_v = cdf_handle.variables['V']
        data_b = cdf_handle.variables['B']
        data_t = cdf_handle.variables['T']
        data_n = cdf_handle.variables['N']
        data_p = cdf_handle.variables['P_dyn']
        data_d = cdf_handle.variables['Delta_angle']
        for time, datum_v, datum_b, datum_t, datum_p, datum_n, datum_d \
                in zip(times, data_v, data_b, data_t, data_n, data_p, data_d):
            vrad = datum_v[0]
            vtan = datum_v[1]
            dtime = datetime_from_list(time)
            if started_at <= dtime <= stopped_at:
                dkey = dtime.strftime(precision)
                x_hci = None
                y_hci = None
                if dkey in orbits_data:
                    x_hci = orbits_data[dkey][0]
                    y_hci = orbits_data[dkey][1]
                all_data[dkey] = (
                    dtime.strftime("%Y-%m-%dT%H:%M:%S+00:00"),
                    vrad, vtan, sqrt(vrad * vrad + vtan * vtan),
                    datum_b, datum_t, datum_n, datum_p, datum_d,
                    x_hci, y_hci
                )
        cdf_handle.close()

    for dkey in sorted(all_data):
        cw.writerow(all_data[dkey])
2d2af24b   Goutte   Add a basic orbit...
337
338
339

    return si.getvalue()

8644387c   Goutte   Use real data.
340

9390ec89   Goutte   Initial experimen...
341
342
343
344
345
346
# MAIN ########################################################################

if __name__ == "__main__":
    # Debug mode on, as the production server does not use this.
    extra_files = [get_path('../config.yml')]
    app.run(debug=True, extra_files=extra_files)