import random
import datetime
import StringIO

from math import sqrt

from os import listdir, environ, remove as removefile

from os.path import isfile, join, abspath, dirname, isdir

import csv

import json
import gzip

import tarfile

import urllib

import logging

from pprint import pprint
from csv import writer as csv_writer
from yaml import load as yaml_load
from flask import Flask

from flask import redirect, url_for, send_from_directory, abort as abort_flask

from flask import request
from jinja2 import Environment, FileSystemLoader
from netCDF4 import Dataset

# PATH RELATIVITY #############################################################

THIS_DIRECTORY = dirname(abspath(__file__))


def get_path(relative_path):

    """Get an absolute path from the relative path to this script directory."""

    return abspath(join(THIS_DIRECTORY, relative_path))


# COLLECT GLOBAL INFORMATION FROM SOURCES #####################################

# VERSION
with open(get_path('../VERSION'), 'r') as version_file:
    version = version_file.read().strip()

# CONFIG
with open(get_path('../config.yml'), 'r') as config_file:
    config = yaml_load(config_file.read())

FILE_DATE_FMT = "%Y-%m-%dT%H:%M:%S"

# LOGGING #####################################################################

log = logging.getLogger("HelioPropa")

log.setLevel(logging.DEBUG)

# log.setLevel(logging.ERROR)                        # <-- set log level here !

logHandler = logging.FileHandler(get_path('run.log'))
logHandler.setFormatter(logging.Formatter(
    "%(asctime)s - %(levelname)s - %(message)s"
))
log.addHandler(logHandler)

# SETUP FLASK ENGINE ##########################################################

app = Flask(__name__, root_path=THIS_DIRECTORY)
app.debug = environ.get('DEBUG') == 'true'

if app.debug:

    log.info("Starting Flask app IN DEBUG MODE...")

else:
    log.info("Starting Flask app...")

# SETUP JINJA2 TEMPLATE ENGINE ################################################

def static_global(filename):
    return url_for('static', filename=filename)


def shuffle_filter(seq):
    """
    This shuffles the sequence it is applied to.

    Jinja2 _should_ provide this.

    """
    try:
        result = list(seq)
        random.shuffle(result)
        return result
    except:
        return seq


def markdown_filter(value, nl2br=False, p=True):
    """

    Converts markdown into html.

    nl2br: set to True to replace line breaks with <br> tags
    p: set to False to remove the enclosing <p></p> tags
    """
    from markdown import markdown
    from markdown.extensions.nl2br import Nl2BrExtension
    from markdown.extensions.abbr import AbbrExtension
    extensions = [AbbrExtension()]
    if nl2br is True:
        extensions.append(Nl2BrExtension())
    markdowned = markdown(value, output_format='html5', extensions=extensions)
    if p is False:
        markdowned = markdowned.replace(r"<p>", "").replace(r"</p>", "")
    return markdowned


tpl_engine = Environment(loader=FileSystemLoader([get_path('view')]),
                         trim_blocks=True,
                         lstrip_blocks=True)

tpl_engine.globals.update(
    url_for=url_for,
    static=static_global,
)

tpl_engine.filters['markdown'] = markdown_filter
tpl_engine.filters['md'] = markdown_filter
tpl_engine.filters['shuffle'] = shuffle_filter

tpl_global_vars = {
    'request': request,
    'version': version,
    'config': config,
    'now': datetime.datetime.now(),
}


# HELPERS #####################################################################

def abort(code, message):
    log.error(message)
    abort_flask(code, message)

def render_view(view, context=None):
    """
    A simple helper to render [view] template with [context] vars.
    It automatically adds the global template vars defined above, too.
    It returns a string, usually the HTML contents to display.
    """
    context = {} if context is None else context
    return tpl_engine.get_template(view).render(
        dict(tpl_global_vars.items() + context.items())
    )


# def render_page(page, title="My Page", context=None):
#     """
#     A simple helper to render the md_page.html template with [context] vars &
#     the additional contents of `page/[page].md` in the `md_page` variable.
#     It automagically adds the global template vars defined above, too.
#     It returns a string, usually the HTML contents to display.
#     """
#     if context is None:
#         context = {}
#     context['title'] = title
#     context['md_page'] = ''
#     with file(get_path('page/%s.md' % page)) as f:
#         context['md_page'] = f.read()
#     return tpl_engine.get_template('md_page.html').render(
#         dict(tpl_global_vars.items() + context.items())
#     )

def datetime_from_list(time_list):

    """

    Datetimes in retrieved CDFs are stored as lists of numbers,

    with DayOfYear starting at 0. We want it starting at 1 for default parsers.
    """

    # Day Of Year starts at 0, but for our datetime parser it starts at 1
    doy = '{:03d}'.format(int(''.join(time_list[4:7])) + 1)
    return datetime.datetime.strptime(
        "%s%s%s" % (''.join(time_list[0:4]), doy, ''.join(time_list[7:])),
        "%Y%j%H%M%S%f"
    )

def get_source_config(slug):

    for s in config['targets']:  # dumb

        if s['slug'] == slug:
            return s
    raise Exception("No source found for slug '%s'." % slug)


def retrieve_data(orbiter, what, started_at, stopped_at):
    """
    Handles remote querying Myriam's API, downloading, extracting and caching
    the netCDF files.
    :param orbiter: key of the source in the YAML config
    :param what: either 'model' or 'orbit', a key in the config of the source
    :param started_at:
    :param stopped_at:
    :return: a list of local file paths to netCDF (.nc) files
    """

    url = config['amda'].format(
        dataSet=what,
        startTime=started_at.isoformat(),
        stopTime=stopped_at.isoformat()
    )

    log.info("Fetching remote gzip files list at '%s'." % url)

    retries = 0
    success = False

    errors = []

    remote_gzip_files = []
    while not success and retries < 3:
        try:
            response = urllib.urlopen(url)
            remote_gzip_files = json.loads(response.read())
            if not remote_gzip_files:
                raise Exception("Failed to fetch data at '%s'." % url)
            if remote_gzip_files == 'NODATASET':

                raise Exception("API says there's no dataset at '%s'." % url)
            if remote_gzip_files == 'ERROR':
                raise Exception("API returned an error at '%s'." % url)

            if remote_gzip_files == ['OUTOFTIME']:  # it happens

                raise Exception("API says it's out of time at '%s'." % url)

            success = True
        except Exception as e:
            log.warn("Failed (%d/3) '%s' : %s" % (retries+1, url, e.message))

            remote_gzip_files = []
            errors.append(e)

        finally:
            retries += 1
    if not remote_gzip_files:

        abort(400, "Failed to fetch gzip files list for %s at '%s' : %s" %
                   (orbiter, url, errors))
    else:
        remote_gzip_files = list(set(remote_gzip_files))

    log.debug("Fetched remote gzip files list : %s." % str(remote_gzip_files))

    local_gzip_files = []
    for remote_gzip_file in remote_gzip_files:

        # hotfixes to remove when fixed upstream @Myriam
        if remote_gzip_file in ['OUTOFTIME', 'ERROR']:
            continue  # sometimes half the response is okay, the other not

        if remote_gzip_file.endswith('/.gz'):
            continue
        remote_gzip_file = remote_gzip_file.replace('cdpp1', 'cdpp', 1)

        ################################################

        filename = "%s_%s" % (orbiter, str(remote_gzip_file).split('/')[-1])
        local_gzip_file = get_path("../cache/%s" % filename)
        local_gzip_files.append(local_gzip_file)
        if not isfile(local_gzip_file):

            log.debug("Retrieving '%s'..." % local_gzip_file)

            urllib.urlretrieve(remote_gzip_file, local_gzip_file)

            log.debug("Retrieved '%s'." % local_gzip_file)

    local_netc_files = []
    for local_gzip_file in local_gzip_files:
        local_netc_file = local_gzip_file[0:-3]
        local_netc_files.append(local_netc_file)

        log.debug("Unzipping '%s'..." % local_gzip_file)

        success = True
        try:
            with gzip.open(local_gzip_file, 'rb') as f:
                file_content = f.read()
                with open(local_netc_file, 'w+b') as g:
                    g.write(file_content)
        except Exception as e:
            success = False
            log.warning("Cannot process gz file '%s' from '%s' : %s" %
                        (local_gzip_file, url, e))
        if success:
            log.debug("Unzipped '%s'." % local_gzip_file)

    return local_netc_files

def generate_csv_contents(source_config, started_at, stopped_at):

    # @todo iterate on models when there are many

    try:
        model_slug = source_config['models'][0]['slug']

    except Exception as e:
        abort(500, "Invalid model configuration for '%s' : %s"
                   % (source_config['slug'], str(e)))

    # Grab the list of netCDF files from Myriam's API
    # http://cdpp.irap.omp.eu/BASE/DDService/getDataUrl.php?dataSet=jupiter_orb_all&StartTime=2014-02-23T10:00:10&StopTime=2017-02-24T23:59:00
    # http://cdpp.irap.omp.eu/BASE/DATA/TAO/JUPITER/SW/sw_2014.nc.gz

    log.info("Generating CSV for '%s'..." % source_config['slug'])

    model_files = retrieve_data(source_config['slug'], model_slug, started_at, stopped_at)
    orbits_files = retrieve_data(source_config['slug'], source_config['orbit']['model'], started_at, stopped_at)

    log.debug("Crunching CSV contents for '%s'..." % source_config['name'])

    si = StringIO.StringIO()
    cw = csv_writer(si)

    cw.writerow((  # the order matters !

        'time',
        'vrad', 'vtan', 'vlen',

        'magn', 'temp', 'pdyn', 'dens', 'angl',
        'xhci', 'yhci'

    ))

    precision = "%Y-%m-%dT%H"  # model and orbits times are equal-ish
    orbits_data = {}  # keys are datetime as str, values arrays of XY
    for orbits_file in orbits_files:

        log.debug("%s: opening orbit NETCDF4 '%s'..." %
                  (source_config['name'], orbits_file))

        cdf_handle = Dataset(orbits_file, "r", format="NETCDF4")
        times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
        data_hci = cdf_handle.variables['HCI']
        for time, datum_hci in zip(times, data_hci):
            dtime = datetime_from_list(time)
            if started_at <= dtime <= stopped_at:
                dkey = dtime.strftime(precision)
                orbits_data[dkey] = datum_hci

        cdf_handle.close()

    all_data = {}  # keys are datetime as str, values tuples of data
    for model_file in model_files:
        # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn

        log.debug("%s: opening model NETCDF4 '%s'..." %
                  (source_config['name'], model_file))

        cdf_handle = Dataset(model_file, "r", format="NETCDF4")
        times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
        data_v = cdf_handle.variables['V']
        data_b = cdf_handle.variables['B']
        data_t = cdf_handle.variables['T']
        data_n = cdf_handle.variables['N']
        data_p = cdf_handle.variables['P_dyn']
        data_d = cdf_handle.variables['Delta_angle']

        log.debug("%s: aggregating data from '%s'..." %
                  (source_config['name'], model_file))

        for time, datum_v, datum_b, datum_t, datum_n, datum_p, datum_d \

                in zip(times, data_v, data_b, data_t, data_n, data_p, data_d):
            vrad = datum_v[0]
            vtan = datum_v[1]
            dtime = datetime_from_list(time)
            if started_at <= dtime <= stopped_at:
                dkey = dtime.strftime(precision)
                x_hci = None
                y_hci = None
                if dkey in orbits_data:
                    x_hci = orbits_data[dkey][0]
                    y_hci = orbits_data[dkey][1]
                all_data[dkey] = (
                    dtime.strftime("%Y-%m-%dT%H:%M:%S+00:00"),
                    vrad, vtan, sqrt(vrad * vrad + vtan * vtan),
                    datum_b, datum_t, datum_n, datum_p, datum_d,
                    x_hci, y_hci
                )
        cdf_handle.close()

    log.debug("Writing and sorting CSV for '%s'..." % source_config['slug'])

    for dkey in sorted(all_data):
        cw.writerow(all_data[dkey])

    log.info("Generated CSV contents for '%s'." % source_config['slug'])

    return si.getvalue()

def generate_csv_file_if_needed(target_config, started_at, stopped_at):

    filename = "%s_%s_%s.csv" % (target_config['slug'],

                                 started_at.strftime(FILE_DATE_FMT),
                                 stopped_at.strftime(FILE_DATE_FMT))
    local_csv_file = get_path("../cache/%s" % filename)
    if not isfile(local_csv_file):
        log.info("Generating CSV '%s'..." % local_csv_file)
        try:
            with open(local_csv_file, mode="w+") as f:
                f.write(generate_csv_contents(target_config,
                                              started_at=started_at,
                                              stopped_at=stopped_at))
            log.info("Generation of '%s' done." % filename)
        except Exception as e:

            if isfile(local_csv_file):

                log.warn("Removing failed CSV '%s'..." % local_csv_file)

                removefile(local_csv_file)

            abort(500, "Failed creating CSV '%s' : %s" % (filename, e))

def remove_files_created_before(date, in_directory):
    """

    Will throw if something horrible happens.
    Does not remove recursively (could be done with os.walk if needed).
    Does not remove directories either.

    :param date: datetime object

    :param in_directory: absolute path to directory

    :return:
    """
    import os
    import time

    secs = time.mktime(date.timetuple())

    if not os.path.isdir(in_directory):
        raise ValueError("No directory to clean at '%s'.")

    removed_files = []
    for file_name in os.listdir(in_directory):
        file_path = os.path.join(in_directory, file_name)

        if os.path.isfile(file_path):
            t = os.stat(file_path)
            if t.st_ctime < secs:
                os.remove(file_path)
                removed_files.append(file_path)

    return removed_files

def get_hit_counter():
    hit_count_path = get_path("../VISITS")

    if isfile(hit_count_path):
        hit_count = int(open(hit_count_path).read())
    else:
        hit_count = 1

    return hit_count

def increment_hit_counter():
    hit_count_path = get_path("../VISITS")

    if isfile(hit_count_path):
        hit_count = int(open(hit_count_path).read())
        hit_count += 1
    else:
        hit_count = 1

    hit_counter_file = open(hit_count_path, 'w')
    hit_counter_file.write(str(hit_count))
    hit_counter_file.close()

    return hit_count

tpl_global_vars['visits'] = get_hit_counter()

# ROUTING #####################################################################

@app.route('/favicon.ico')
def favicon():
    return send_from_directory(
        join(app.root_path, 'static', 'img'),
        'favicon.ico', mimetype='image/vnd.microsoft.icon'
    )


@app.route("/")
@app.route("/home.html")
@app.route("/index.html")
def home():

    increment_hit_counter()

    return render_view('home.html.jinja2', {
        'targets': config['targets'],
        'planets': [s for s in config['targets'] if s['type'] == 'planet'],
        'probes':  [s for s in config['targets'] if s['type'] == 'probe'],
        'comets':  [s for s in config['targets'] if s['type'] == 'comet'],

        'visits':  get_hit_counter(),

    })


@app.route("/<source>_<started_at>_<stopped_at>.csv")
def get_target_csv(source, started_at, stopped_at):
    """
    Grab data and orbit data for the specified `target`,
    rearrange it and return it as a CSV file.
    `started_at` and `stopped_at` should be UTC.
    """

    source_config = get_source_config(source)

    try:

        started_at = datetime.datetime.strptime(started_at, FILE_DATE_FMT)

    except:
        abort(400, "Invalid started_at parameter : '%s'." % started_at)
    try:

        stopped_at = datetime.datetime.strptime(stopped_at, FILE_DATE_FMT)

    except:
        abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at)

    filename = "%s_%s_%s.csv" % (source,

                                 started_at.strftime(FILE_DATE_FMT),
                                 stopped_at.strftime(FILE_DATE_FMT))

    local_csv_file = get_path("../cache/%s" % filename)

    generate_csv_file_if_needed(source_config, started_at, stopped_at)

    if not isfile(local_csv_file):
        abort(500, "Could not cache CSV file at '%s'." % local_csv_file)

    return send_from_directory(get_path("../cache/"), filename)

@app.route("/<targets>_<started_at>_<stopped_at>.tar.gz")
def download_targets_tarball(targets, started_at, stopped_at):

    """
    Grab data and orbit data for the specified `target`,
    rearrange it and return it as a CSV file.
    `started_at` and `stopped_at` should be UTC.

    targets: string list of targets' slugs, separated by `-`.
    This will fail hard if targets' slugs start having `-` in them.

    """

    separator = '-'

    targets = targets.split(separator)
    targets.sort()

    targets_configs = []
    for target in targets:

        if not target:
            abort(400, "Invalid targets format : `%s`." % targets)

        targets_configs.append(get_source_config(target))
    if 0 == len(targets_configs):

        abort(400, "No valid targets specified. What are you doing?")

    date_fmt = "%Y-%m-%dT%H:%M:%S"
    try:
        started_at = datetime.datetime.strptime(started_at, date_fmt)
    except:
        abort(400, "Invalid started_at parameter : '%s'." % started_at)
    try:
        stopped_at = datetime.datetime.strptime(stopped_at, date_fmt)
    except:
        abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at)

    sta = started_at.strftime(date_fmt)
    sto = stopped_at.strftime(date_fmt)

    gzip_filename = "%s_%s_%s.tar.gz" % (separator.join(targets), sta, sto)

    local_gzip_file = get_path("../cache/%s" % gzip_filename)

    if not isfile(local_gzip_file):

        log.debug("Creating the CSV files for the tarball...")

        for target_config in targets_configs:

            filename = "%s_%s_%s.csv" % (target_config['slug'], sta, sto)

            local_csv_file = get_path("../cache/%s" % filename)
            if not isfile(local_csv_file):
                with open(local_csv_file, mode="w+") as f:
                    f.write(generate_csv_contents(target_config,
                                                  started_at=started_at,
                                                  stopped_at=stopped_at))

        log.debug("Creating the tarball '%s'..." % local_gzip_file)

        with tarfile.open(local_gzip_file, "w:gz") as tar:
            for target_config in targets_configs:

                filename = "%s_%s_%s.csv" % (target_config['slug'], sta, sto)

                local_csv_file = get_path("../cache/%s" % filename)
                tar.add(local_csv_file, arcname=filename)

    if not isfile(local_gzip_file):

        abort(500, "No tarball to serve. Looked at '%s'." % local_gzip_file)

    return send_from_directory(get_path("../cache/"), gzip_filename)

# API #########################################################################

@app.route("/cache/clear")
def cache_clear():
    """
    Removes all files from the cache that are older than roughly one month.
    """
    a_month_ago = datetime.datetime.now() - datetime.timedelta(days=32)
    cache_dir = get_path('../cache')
    removed_files = remove_files_created_before(a_month_ago, cache_dir)
    return "Cache cleared! Removed %d old file(s)." % len(removed_files)

# DEV TOOLS ###################################################################

# @app.route("/inspect")
# def analyze_cdf():

#     """

#     For debug purposes.

#     """

#     cdf_to_inspect = get_path("../res/dummy.nc")
#     cdf_to_inspect = get_path("../res/dummy_jupiter_coordinates.nc")

#
#     si = StringIO.StringIO()

#     cw = csv.DictWriter(si, fieldnames=['Name', 'Shape', 'Length'])
#     cw.writeheader()

#

#     # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn, QualityFlag
#     cdf_handle = Dataset(cdf_to_inspect, "r", format="NETCDF4")
#     for variable in cdf_handle.variables:
#         v = cdf_handle.variables[variable]
#         cw.writerow({
#             'Name': variable,
#             'Shape': v.shape,
#             'Length': v.size,
#         })
#     cdf_handle.close()

#
#     return si.getvalue()

# MAIN ########################################################################

if __name__ == "__main__":
    # Debug mode on, as the production server does not use this.
    extra_files = [get_path('../config.yml')]
    app.run(debug=True, extra_files=extra_files)