Blame view

web/run.py 16.3 KB
9390ec89   Goutte   Initial experimen...
1
2
3
import random
import datetime
import StringIO
c42fea3a   Goutte   Rework the CSS wi...
4
from math import sqrt
9390ec89   Goutte   Initial experimen...
5
6
7
8
9

from os import listdir, environ
from os.path import isfile, join, abspath, dirname

import csv
8644387c   Goutte   Use real data.
10
11
12
import json
import gzip
import urllib
f75faf5f   Goutte   WIP
13
import logging
9390ec89   Goutte   Initial experimen...
14
15
16
17
from pprint import pprint
from csv import writer as csv_writer
from yaml import load as yaml_load
from flask import Flask
57f42bd7   Goutte   Log the abortions.
18
from flask import redirect, url_for, send_from_directory, abort as abort_flask
9390ec89   Goutte   Initial experimen...
19
20
21
22
from flask import request
from jinja2 import Environment, FileSystemLoader
from netCDF4 import Dataset

9390ec89   Goutte   Initial experimen...
23
24
25
26
27
28
29

# PATH RELATIVITY #############################################################

THIS_DIRECTORY = dirname(abspath(__file__))


def get_path(relative_path):
a4a9ef03   Goutte   Cache generated C...
30
    """Get an absolute path from the relative path to this script directory."""
9390ec89   Goutte   Initial experimen...
31
32
33
34
35
36
37
38
39
40
41
42
43
    return abspath(join(THIS_DIRECTORY, relative_path))


# COLLECT GLOBAL INFORMATION FROM SOURCES #####################################

# VERSION
with open(get_path('../VERSION'), 'r') as version_file:
    version = version_file.read().strip()

# CONFIG
with open(get_path('../config.yml'), 'r') as config_file:
    config = yaml_load(config_file.read())

c0df94bc   Goutte   Adding more logs.
44
45
FILE_DATE_FMT = "%Y-%m-%dT%H:%M:%S"

9390ec89   Goutte   Initial experimen...
46

f75faf5f   Goutte   WIP
47
48
49
50
# LOGGING #####################################################################

log = logging.getLogger("HelioPropa")
log.setLevel(logging.INFO)
b2837a08   Goutte   Add three retries...
51
52
53
54
55
logHandler = logging.FileHandler(get_path('run.log'))
logHandler.setFormatter(logging.Formatter(
    "%(asctime)s - %(levelname)s - %(message)s"
))
log.addHandler(logHandler)
f75faf5f   Goutte   WIP
56
57


9390ec89   Goutte   Initial experimen...
58
59
60
61
# SETUP FLASK ENGINE ##########################################################

app = Flask(__name__, root_path=THIS_DIRECTORY)
app.debug = environ.get('DEBUG') == 'true'
b2837a08   Goutte   Add three retries...
62
63
64
65
if app.debug:
    log.info("Starting Flask app in debug mode...")
else:
    log.info("Starting Flask app...")
9390ec89   Goutte   Initial experimen...
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126


# SETUP JINJA2 TEMPLATE ENGINE ################################################

def static_global(filename):
    return url_for('static', filename=filename)


def shuffle_filter(seq):
    """
    This shuffles the sequence it is applied to.
    'tis a failure of jinja2 to not provide a shuffle filter by default.
    """
    try:
        result = list(seq)
        random.shuffle(result)
        return result
    except:
        return seq


def markdown_filter(value, nl2br=False, p=True):
    """
    nl2br: set to True to replace line breaks with <br> tags
    p: set to False to remove the enclosing <p></p> tags
    """
    from markdown import markdown
    from markdown.extensions.nl2br import Nl2BrExtension
    from markdown.extensions.abbr import AbbrExtension
    extensions = [AbbrExtension()]
    if nl2br is True:
        extensions.append(Nl2BrExtension())
    markdowned = markdown(value, output_format='html5', extensions=extensions)
    if p is False:
        markdowned = markdowned.replace(r"<p>", "").replace(r"</p>", "")
    return markdowned


tpl_engine = Environment(loader=FileSystemLoader([get_path('view')]),
                         trim_blocks=True,
                         lstrip_blocks=True)

tpl_engine.globals.update(
    url_for=url_for,
    static=static_global,
)

tpl_engine.filters['markdown'] = markdown_filter
tpl_engine.filters['md'] = markdown_filter
tpl_engine.filters['shuffle'] = shuffle_filter

tpl_global_vars = {
    'request': request,
    'version': version,
    'config': config,
    'now': datetime.datetime.now(),
}


# HELPERS #####################################################################

57f42bd7   Goutte   Log the abortions.
127
128
129
130
131
def abort(code, message):
    log.error(message)
    abort_flask(code, message)


9390ec89   Goutte   Initial experimen...
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
def render_view(view, context=None):
    """
    A simple helper to render [view] template with [context] vars.
    It automatically adds the global template vars defined above, too.
    It returns a string, usually the HTML contents to display.
    """
    context = {} if context is None else context
    return tpl_engine.get_template(view).render(
        dict(tpl_global_vars.items() + context.items())
    )


# def render_page(page, title="My Page", context=None):
#     """
#     A simple helper to render the md_page.html template with [context] vars &
#     the additional contents of `page/[page].md` in the `md_page` variable.
#     It automagically adds the global template vars defined above, too.
#     It returns a string, usually the HTML contents to display.
#     """
#     if context is None:
#         context = {}
#     context['title'] = title
#     context['md_page'] = ''
#     with file(get_path('page/%s.md' % page)) as f:
#         context['md_page'] = f.read()
#     return tpl_engine.get_template('md_page.html').render(
#         dict(tpl_global_vars.items() + context.items())
#     )

2d2af24b   Goutte   Add a basic orbit...
161
def datetime_from_list(time_list):
0b9821dd   Goutte   Clean up.
162
163
164
165
    """
    Datetimes in retrieved CDFs are stored in lists of numbers,
    with DayOfYear starting at 0. We want it starting at 1 for default parsers.
    """
2d2af24b   Goutte   Add a basic orbit...
166
167
168
169
170
171
    # Day Of Year starts at 0, but for our datetime parser it starts at 1
    doy = '{:03d}'.format(int(''.join(time_list[4:7])) + 1)
    return datetime.datetime.strptime(
        "%s%s%s" % (''.join(time_list[0:4]), doy, ''.join(time_list[7:])),
        "%Y%j%H%M%S%f"
    )
9390ec89   Goutte   Initial experimen...
172

ce8af118   Goutte   Fix the favicon.
173

8644387c   Goutte   Use real data.
174
def get_source_config(slug):
7d6dee0f   Goutte   Continue refacto ...
175
    for s in config['targets']:
8644387c   Goutte   Use real data.
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
        if s['slug'] == slug:
            return s
    raise Exception("No source found for slug '%s'." % slug)


def retrieve_data(orbiter, what, started_at, stopped_at):
    """
    Handles remote querying Myriam's API, downloading, extracting and caching
    the netCDF files.
    :param orbiter: key of the source in the YAML config
    :param what: either 'model' or 'orbit', a key in the config of the source
    :param started_at:
    :param stopped_at:
    :return: a list of local file paths to netCDF (.nc) files
    """

    url = config['amda'].format(
        dataSet=what,
        startTime=started_at.isoformat(),
        stopTime=stopped_at.isoformat()
    )
b2837a08   Goutte   Add three retries...
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
    retries = 0
    success = False
    remote_gzip_files = []
    while not success and retries < 3:
        try:
            response = urllib.urlopen(url)
            remote_gzip_files = json.loads(response.read())
            if not remote_gzip_files:
                raise Exception("Failed to fetch data at '%s'." % url)
            if remote_gzip_files == 'NODATASET':
                raise Exception("No dataset at '%s'." % url)
            success = True
        except Exception as e:
            log.warn("Failed (%d/3) '%s' : %s" % (retries+1, url, e.message))
        finally:
            retries += 1
    if not remote_gzip_files:
8644387c   Goutte   Use real data.
214
        abort(400, "Failed to fetch data at '%s'." % url)
b2837a08   Goutte   Add three retries...
215
216
    if remote_gzip_files == 'NODATASET':
        abort(400, "No dataset at '%s'." % url)
8644387c   Goutte   Use real data.
217
218
219
220

    # retriever = urllib.URLopener()  # would we need to do this every time ?
    local_gzip_files = []
    for remote_gzip_file in remote_gzip_files:
4cf497e0   Goutte   Make the targets ...
221
        # hotfix removeme @Myriam
8644387c   Goutte   Use real data.
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
        if remote_gzip_file.endswith('/.gz'):
            continue
        remote_gzip_file = remote_gzip_file.replace('cdpp1', 'cdpp', 1)
        #########################
        filename = "%s_%s" % (orbiter, str(remote_gzip_file).split('/')[-1])
        local_gzip_file = get_path("../cache/%s" % filename)
        local_gzip_files.append(local_gzip_file)
        if not isfile(local_gzip_file):
            urllib.urlretrieve(remote_gzip_file, local_gzip_file)

    local_netc_files = []
    for local_gzip_file in local_gzip_files:
        local_netc_file = local_gzip_file[0:-3]
        local_netc_files.append(local_netc_file)
        with gzip.open(local_gzip_file, 'rb') as f:
            file_content = f.read()
            with open(local_netc_file, 'w+b') as g:
                g.write(file_content)

    return local_netc_files


a4a9ef03   Goutte   Cache generated C...
244
def generate_csv_contents(source_config, started_at, stopped_at):
b2837a08   Goutte   Add three retries...
245
    # @todo iterate on models when there are many
8644387c   Goutte   Use real data.
246
247
248
    try:
        model_slug = source_config['models'][0]['slug']
    except:
a4a9ef03   Goutte   Cache generated C...
249
        abort(500, "Invalid model configuration for '%s'." % source_config['slug'])
28ef3790   Goutte   Clean up.
250
251
252
253

    # Grab the list of netCDF files from Myriam's API
    # http://cdpp.irap.omp.eu/BASE/DDService/getDataUrl.php?dataSet=jupiter_orb_all&StartTime=2014-02-23T10:00:10&StopTime=2017-02-24T23:59:00
    # http://cdpp.irap.omp.eu/BASE/DATA/TAO/JUPITER/SW/sw_2014.nc.gz
a4a9ef03   Goutte   Cache generated C...
254
255
    model_files = retrieve_data(source_config['slug'], model_slug, started_at, stopped_at)
    orbits_files = retrieve_data(source_config['slug'], source_config['orbit']['model'], started_at, stopped_at)
61179cdc   Goutte   Initial work on t...
256
257
258

    si = StringIO.StringIO()
    cw = csv_writer(si)
8644387c   Goutte   Use real data.
259
    cw.writerow((  # the order matters !
61179cdc   Goutte   Initial work on t...
260
261
        'time',
        'vrad', 'vtan', 'vlen',
8644387c   Goutte   Use real data.
262
263
        'magn', 'temp', 'pdyn', 'dens', 'angl',
        'xhci', 'yhci'
61179cdc   Goutte   Initial work on t...
264
    ))
9390ec89   Goutte   Initial experimen...
265

8644387c   Goutte   Use real data.
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
    precision = "%Y-%m-%dT%H"  # model and orbits times are equal-ish
    orbits_data = {}  # keys are datetime as str, values arrays of XY
    for orbits_file in orbits_files:
        cdf_handle = Dataset(orbits_file, "r", format="NETCDF4")
        times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
        data_hci = cdf_handle.variables['HCI']
        for time, datum_hci in zip(times, data_hci):
            dtime = datetime_from_list(time)
            if started_at <= dtime <= stopped_at:
                dkey = dtime.strftime(precision)
                orbits_data[dkey] = datum_hci
    all_data = {}  # keys are datetime as str, values tuples of data
    for model_file in model_files:
        # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn
        cdf_handle = Dataset(model_file, "r", format="NETCDF4")
        times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
        data_v = cdf_handle.variables['V']
        data_b = cdf_handle.variables['B']
        data_t = cdf_handle.variables['T']
        data_n = cdf_handle.variables['N']
        data_p = cdf_handle.variables['P_dyn']
        data_d = cdf_handle.variables['Delta_angle']
        for time, datum_v, datum_b, datum_t, datum_p, datum_n, datum_d \
                in zip(times, data_v, data_b, data_t, data_n, data_p, data_d):
            vrad = datum_v[0]
            vtan = datum_v[1]
            dtime = datetime_from_list(time)
            if started_at <= dtime <= stopped_at:
                dkey = dtime.strftime(precision)
                x_hci = None
                y_hci = None
                if dkey in orbits_data:
                    x_hci = orbits_data[dkey][0]
                    y_hci = orbits_data[dkey][1]
                all_data[dkey] = (
                    dtime.strftime("%Y-%m-%dT%H:%M:%S+00:00"),
                    vrad, vtan, sqrt(vrad * vrad + vtan * vtan),
                    datum_b, datum_t, datum_n, datum_p, datum_d,
                    x_hci, y_hci
                )
        cdf_handle.close()

    for dkey in sorted(all_data):
        cw.writerow(all_data[dkey])
2d2af24b   Goutte   Add a basic orbit...
310
311
312

    return si.getvalue()

8644387c   Goutte   Use real data.
313

c0df94bc   Goutte   Adding more logs.
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
def generate_csv_file_if_needed(target_config, started_at, stopped_at):
    filename = "%s_%s_%s.csv" % (target_config.slug,
                                 started_at.strftime(FILE_DATE_FMT),
                                 stopped_at.strftime(FILE_DATE_FMT))
    local_csv_file = get_path("../cache/%s" % filename)
    if not isfile(local_csv_file):
        log.info("Generating CSV '%s'..." % local_csv_file)
        try:
            with open(local_csv_file, mode="w+") as f:
                f.write(generate_csv_contents(target_config,
                                              started_at=started_at,
                                              stopped_at=stopped_at))
            log.info("Generation of '%s' done." % filename)
        except Exception as e:
            abort(500, "Failed creating CSV '%s' : %s" % (filename, e.message))


a4a9ef03   Goutte   Cache generated C...
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
def increment_hit_counter():
    hit_count_path = get_path("../VISITS")

    if isfile(hit_count_path):
        hit_count = int(open(hit_count_path).read())
        hit_count += 1
    else:
        hit_count = 1

    hit_counter_file = open(hit_count_path, 'w')
    hit_counter_file.write(str(hit_count))
    hit_counter_file.close()

    return hit_count


# ROUTING #####################################################################

@app.route('/favicon.ico')
def favicon():
    return send_from_directory(
        join(app.root_path, 'static', 'img'),
        'favicon.ico', mimetype='image/vnd.microsoft.icon'
    )


@app.route("/")
@app.route("/home.html")
@app.route("/index.html")
def home():
    return render_view('home.html.jinja2', {
        'targets': config['targets'],
        'planets': [s for s in config['targets'] if s['type'] == 'planet'],
        'probes':  [s for s in config['targets'] if s['type'] == 'probe'],
        'comets':  [s for s in config['targets'] if s['type'] == 'comet'],
        'visits':  increment_hit_counter(),
    })


@app.route("/<source>_<started_at>_<stopped_at>.csv")
def get_target_csv(source, started_at, stopped_at):
    """
    Grab data and orbit data for the specified `target`,
    rearrange it and return it as a CSV file.
    `started_at` and `stopped_at` should be UTC.
    """
a4a9ef03   Goutte   Cache generated C...
377
    source_config = get_source_config(source)
a4a9ef03   Goutte   Cache generated C...
378
    try:
c0df94bc   Goutte   Adding more logs.
379
        started_at = datetime.datetime.strptime(started_at, FILE_DATE_FMT)
a4a9ef03   Goutte   Cache generated C...
380
381
382
    except:
        abort(400, "Invalid started_at parameter : '%s'." % started_at)
    try:
c0df94bc   Goutte   Adding more logs.
383
        stopped_at = datetime.datetime.strptime(stopped_at, FILE_DATE_FMT)
a4a9ef03   Goutte   Cache generated C...
384
385
386
    except:
        abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at)

a4a9ef03   Goutte   Cache generated C...
387
    filename = "%s_%s_%s.csv" % (source,
c0df94bc   Goutte   Adding more logs.
388
389
                                 started_at.strftime(FILE_DATE_FMT),
                                 stopped_at.strftime(FILE_DATE_FMT))
a4a9ef03   Goutte   Cache generated C...
390
    local_csv_file = get_path("../cache/%s" % filename)
c0df94bc   Goutte   Adding more logs.
391
    generate_csv_file_if_needed(source_config, started_at, stopped_at)
a4a9ef03   Goutte   Cache generated C...
392
393
394
395
396
397
    if not isfile(local_csv_file):
        abort(500, "Could not cache CSV file at '%s'." % local_csv_file)

    return send_from_directory(get_path("../cache/"), filename)


b2837a08   Goutte   Add three retries...
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
@app.route("/<targets>_<started_at>_<stopped_at>.zip")
def download_targets_zip(targets, started_at, stopped_at):
    """
    Grab data and orbit data for the specified `target`,
    rearrange it and return it as a CSV file.
    `started_at` and `stopped_at` should be UTC.

    targets: string list of targets' slugs, separated by `:`.


    fixme


    """

    targets_confs = []
    for target in targets.split(':').sort():
        if not target:
            abort(400, "Invalid targets format : `%s`." % targets)
        targets_confs.append(get_source_config(target))
    if 0 == len(targets_confs):
        abort(400, "No valid targets specified. What are you doing?")

    date_fmt = "%Y-%m-%dT%H:%M:%S"
    try:
        started_at = datetime.datetime.strptime(started_at, date_fmt)
    except:
        abort(400, "Invalid started_at parameter : '%s'." % started_at)
    try:
        stopped_at = datetime.datetime.strptime(stopped_at, date_fmt)
    except:
        abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at)


    filename = "%s_%s_%s.csv" % (source,
                                 started_at.strftime(date_fmt),
                                 stopped_at.strftime(date_fmt))

    local_csv_file = get_path("../cache/%s" % filename)
    if not isfile(local_csv_file):
        with open(local_csv_file, mode="w+") as f:
            f.write(generate_csv_contents(source_config,
                                          started_at=started_at,
                                          stopped_at=stopped_at))

    if not isfile(local_csv_file):
        abort(500, "Could not cache CSV file at '%s'." % local_csv_file)

    return send_from_directory(get_path("../cache/"), filename)

1754789b   Goutte   Decorate and clea...
448
449
450
451
# DEV TOOLS ###################################################################

# @app.route("/inspect")
# def analyze_cdf():
a4a9ef03   Goutte   Cache generated C...
452
#     """
1754789b   Goutte   Decorate and clea...
453
#     For debug purposes.
a4a9ef03   Goutte   Cache generated C...
454
#     """
1754789b   Goutte   Decorate and clea...
455
456
#     cdf_to_inspect = get_path("../res/dummy.nc")
#     cdf_to_inspect = get_path("../res/dummy_jupiter_coordinates.nc")
a4a9ef03   Goutte   Cache generated C...
457
458
#
#     si = StringIO.StringIO()
1754789b   Goutte   Decorate and clea...
459
460
#     cw = csv.DictWriter(si, fieldnames=['Name', 'Shape', 'Length'])
#     cw.writeheader()
a4a9ef03   Goutte   Cache generated C...
461
#
1754789b   Goutte   Decorate and clea...
462
463
464
465
466
467
468
469
470
471
#     # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn, QualityFlag
#     cdf_handle = Dataset(cdf_to_inspect, "r", format="NETCDF4")
#     for variable in cdf_handle.variables:
#         v = cdf_handle.variables[variable]
#         cw.writerow({
#             'Name': variable,
#             'Shape': v.shape,
#             'Length': v.size,
#         })
#     cdf_handle.close()
a4a9ef03   Goutte   Cache generated C...
472
473
474
475
#
#     return si.getvalue()


9390ec89   Goutte   Initial experimen...
476
477
478
479
480
481
# MAIN ########################################################################

if __name__ == "__main__":
    # Debug mode on, as the production server does not use this.
    extra_files = [get_path('../config.yml')]
    app.run(debug=True, extra_files=extra_files)