import StringIO
import datetime
import time
import gzip
import json
import logging
import random
import tarfile
import urllib
from csv import writer as csv_writer
from math import sqrt
from os import environ, remove as removefile
from os.path import isfile, join, abspath, dirname

from flask import Flask
from flask import request
from flask import url_for, send_from_directory, abort as abort_flask
from jinja2 import Environment, FileSystemLoader
from netCDF4 import Dataset
from yaml import load as yaml_load

# PATH RELATIVITY #############################################################

THIS_DIRECTORY = dirname(abspath(__file__))


def get_path(relative_path):
    """Get an absolute path from the relative path to this script directory."""
    return abspath(join(THIS_DIRECTORY, relative_path))


# COLLECT GLOBAL INFORMATION FROM SOURCES #####################################

# VERSION
with open(get_path('../VERSION'), 'r') as version_file:
    version = version_file.read().strip()

# CONFIG
with open(get_path('../config.yml'), 'r') as config_file:
    config = yaml_load(config_file.read())

FILE_DATE_FMT = "%Y-%m-%dT%H:%M:%S"


# LOGGING #####################################################################

log = logging.getLogger("HelioPropa")
log.setLevel(logging.DEBUG)
# log.setLevel(logging.ERROR)                        # <-- set log level here !
logHandler = logging.FileHandler(get_path('run.log'))
logHandler.setFormatter(logging.Formatter(
    "%(asctime)s - %(levelname)s - %(message)s"
))
log.addHandler(logHandler)


# SETUP FLASK ENGINE ##########################################################

app = Flask(__name__, root_path=THIS_DIRECTORY)
app.debug = environ.get('DEBUG') == 'true'
if app.debug:
    log.info("Starting Flask app IN DEBUG MODE...")
else:
    log.info("Starting Flask app...")


# SETUP JINJA2 TEMPLATE ENGINE ################################################

def static_global(filename):
    return url_for('static', filename=filename)


def shuffle_filter(seq):
    """
    This shuffles the sequence it is applied to.
    Jinja2 _should_ provide this.
    """
    try:
        result = list(seq)
        random.shuffle(result)
        return result
    except:
        return seq


def markdown_filter(value, nl2br=False, p=True):
    """
    Converts markdown into html.
    nl2br: set to True to replace line breaks with <br> tags
    p: set to False to remove the enclosing <p></p> tags
    """
    from markdown import markdown
    from markdown.extensions.nl2br import Nl2BrExtension
    from markdown.extensions.abbr import AbbrExtension
    extensions = [AbbrExtension()]
    if nl2br is True:
        extensions.append(Nl2BrExtension())
    markdowned = markdown(value, output_format='html5', extensions=extensions)
    if p is False:
        markdowned = markdowned.replace(r"<p>", "").replace(r"</p>", "")
    return markdowned


tpl_engine = Environment(loader=FileSystemLoader([get_path('view')]),
                         trim_blocks=True,
                         lstrip_blocks=True)

tpl_engine.globals.update(
    url_for=url_for,
    static=static_global,
)

tpl_engine.filters['markdown'] = markdown_filter
tpl_engine.filters['md'] = markdown_filter
tpl_engine.filters['shuffle'] = shuffle_filter

tpl_global_vars = {
    'request': request,
    'version': version,
    'config': config,
    'now': datetime.datetime.now(),
}


# HELPERS #####################################################################

def abort(code, message):
    log.error(message)
    abort_flask(code, message)


def render_view(view, context=None):
    """
    A simple helper to render [view] template with [context] vars.
    It automatically adds the global template vars defined above, too.
    It returns a string, usually the HTML contents to display.
    """
    context = {} if context is None else context
    return tpl_engine.get_template(view).render(
        dict(tpl_global_vars.items() + context.items())
    )


# def render_page(page, title="My Page", context=None):
#     """
#     A simple helper to render the md_page.html template with [context] vars &
#     the additional contents of `page/[page].md` in the `md_page` variable.
#     It automagically adds the global template vars defined above, too.
#     It returns a string, usually the HTML contents to display.
#     """
#     if context is None:
#         context = {}
#     context['title'] = title
#     context['md_page'] = ''
#     with file(get_path('page/%s.md' % page)) as f:
#         context['md_page'] = f.read()
#     return tpl_engine.get_template('md_page.html').render(
#         dict(tpl_global_vars.items() + context.items())
#     )


def is_list_in_list(needle, haystack):
    for n in needle:
        if n not in haystack:
            return False
    return True


def datetime_from_list(time_list):
    """
    Datetimes in retrieved CDFs are stored as lists of numbers,
    with DayOfYear starting at 0. We want it starting at 1 for default parsers.
    """
    # Day Of Year starts at 0, but for our datetime parser it starts at 1
    doy = '{:03d}'.format(int(''.join(time_list[4:7])) + 1)
    return datetime.datetime.strptime(
        "%s%s%s" % (''.join(time_list[0:4]), doy, ''.join(time_list[7:])),
        "%Y%j%H%M%S%f"
    )


def get_target_config(slug):
    for s in config['targets']:  # dumb
        if s['slug'] == slug:
            return s
    raise Exception("No target found in configuration for '%s'." % slug)


def check_target_config(slug):
    get_target_config(slug)


def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at):
    """
    Handles remote querying Myriam's API, downloading, extracting and caching
    the netCDF files.
    :param orbiter: key of the source in the YAML config
    :param what: either 'model' or 'orbit', a key in the config of the source
    :param started_at:
    :param stopped_at:
    :return: a list of local file paths to netCDF (.nc) files
    """

    url = config['amda'].format(
        dataSet=what,
        startTime=started_at.isoformat(),
        stopTime=stopped_at.isoformat()
    )
    log.info("Fetching remote gzip files list at '%s'." % url)
    retries = 0
    success = False
    errors = []
    remote_gzip_files = []
    while not success and retries < 3:
        try:
            response = urllib.urlopen(url)
            remote_gzip_files = json.loads(response.read())
            if not remote_gzip_files:
                raise Exception("Failed to fetch data at '%s'." % url)
            if remote_gzip_files == 'NODATASET':
                raise Exception("API says there's no dataset at '%s'." % url)
            if remote_gzip_files == 'ERROR':
                raise Exception("API returned an error at '%s'." % url)
            if remote_gzip_files == ['OUTOFTIME']:  # it happens
                raise Exception("API says it's out of time at '%s'." % url)
            success = True
        except Exception as e:
            log.warn("Failed (%d/3) '%s' : %s" % (retries+1, url, e.message))
            remote_gzip_files = []
            errors.append(e)
        finally:
            retries += 1
    if not remote_gzip_files:
        abort(400, "Failed to fetch gzip files list for %s at '%s' : %s" %
                   (orbiter, url, errors))
    else:
        remote_gzip_files = list(set(remote_gzip_files))

    log.debug("Fetched remote gzip files list : %s." % str(remote_gzip_files))

    local_gzip_files = []
    for remote_gzip_file in remote_gzip_files:
        # hotfixes to remove when fixed upstream @Myriam
        if remote_gzip_file in ['OUTOFTIME', 'ERROR']:
            continue  # sometimes half the response is okay, the other not
        if remote_gzip_file.endswith('/.gz'):
            continue
        remote_gzip_file = remote_gzip_file.replace('cdpp1', 'cdpp', 1)
        ################################################
        filename = "%s_%s" % (orbiter, str(remote_gzip_file).split('/')[-1])
        local_gzip_file = get_path("../cache/%s" % filename)
        local_gzip_files.append(local_gzip_file)
        if not isfile(local_gzip_file):
            log.debug("Retrieving '%s'..." % local_gzip_file)
            urllib.urlretrieve(remote_gzip_file, local_gzip_file)
            log.debug("Retrieved '%s'." % local_gzip_file)

    local_netc_files = []
    for local_gzip_file in local_gzip_files:
        local_netc_file = local_gzip_file[0:-3]
        local_netc_files.append(local_netc_file)
        log.debug("Unzipping '%s'..." % local_gzip_file)
        success = True
        try:
            with gzip.open(local_gzip_file, 'rb') as f:
                file_content = f.read()
                with open(local_netc_file, 'w+b') as g:
                    g.write(file_content)
        except Exception as e:
            success = False
            log.warning("Cannot process gz file '%s' from '%s' : %s" %
                        (local_gzip_file, url, e))
        if success:
            log.debug("Unzipped '%s'." % local_gzip_file)

    return local_netc_files


# The available parameters in the generated CSV and NetCDF files.
# The order matters. If you change this you also need to change the
# innermost loop of `get_data_for_target`.
# The javascript knows the targets' properties under these names.
# DEPRECATED, use PARAMETERS instead
PROPERTIES = ('time', 'vrad', 'vtan', 'vlen', 'magn', 'temp', 'pdyn', 'dens',
              'angl', 'xhci', 'yhci')

PARAMETERS = {
    'vlen': {
        'slug': 'vlen',
        'name': 'Velocity',
        'unit': 'km/s',
        'show': True,
    },
    'magn': {
        'slug': 'magn',
        'name': 'Magnetism',
        'unit': 'nT',
        'show': True,
    },
    # fixme: add more
}


def get_data_for_target(target_config, started_at, stopped_at):
    """
    :return: dict whose keys are datetime as str, values tuples of data
    """
    log.debug("Grabbing data for '%s'..." % target_config['slug'])
    # @todo iterate on models when there are many
    try:
        model_slug = target_config['models'][0]['slug']
    except Exception as e:
        abort(500, "Invalid model configuration for '%s' : %s"
              % (target_config['slug'], str(e)))
    try:
        orbit_slug = target_config['orbit']['model']
    except Exception as e:
        abort(500, "Invalid orbit configuration for '%s' : %s"
              % (target_config['slug'], str(e)))

    # Grab the list of netCDF files from Myriam's API
    # http://cdpp.irap.omp.eu/BASE/DDService/getDataUrl.php?dataSet=jupiter_orb_all&StartTime=2014-02-23T10:00:10&StopTime=2017-02-24T23:59:00
    # http://cdpp.irap.omp.eu/BASE/DATA/TAO/JUPITER/SW/sw_2014.nc.gz
    model_files = retrieve_amda_netcdf(target_config['slug'], model_slug,
                                       started_at, stopped_at)
    orbit_files = retrieve_amda_netcdf(target_config['slug'], orbit_slug,
                                       started_at, stopped_at)

    precision = "%Y-%m-%dT%H"  # model and orbits times are equal-ish
    orbit_data = {}  # keys are datetime as str, values arrays of XY
    for orbit_file in orbit_files:
        log.debug("%s: opening orbit NETCDF4 '%s'..." %
                  (target_config['name'], orbit_file))
        cdf_handle = Dataset(orbit_file, "r", format="NETCDF4")
        times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
        data_hci = cdf_handle.variables['HCI']
        for time, datum_hci in zip(times, data_hci):
            dtime = datetime_from_list(time)
            if started_at <= dtime <= stopped_at:
                dkey = dtime.strftime(precision)
                orbit_data[dkey] = datum_hci
        cdf_handle.close()

    all_data = {}  # keys are datetime as str, values tuples of data
    for model_file in model_files:
        # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn
        log.debug("%s: opening model NETCDF4 '%s'..." %
                  (target_config['name'], model_file))
        cdf_handle = Dataset(model_file, "r", format="NETCDF4")
        times = cdf_handle.variables['Time']  # YYYY DOY HH MM SS .ms
        data_v = cdf_handle.variables['V']
        data_b = cdf_handle.variables['B']
        data_t = cdf_handle.variables['T']
        data_n = cdf_handle.variables['N']
        data_p = cdf_handle.variables['P_dyn']
        data_d = cdf_handle.variables['Delta_angle']
        log.debug("%s: aggregating data from '%s'..." %
                  (target_config['name'], model_file))
        for time, datum_v, datum_b, datum_t, datum_n, datum_p, datum_d \
                in zip(times, data_v, data_b, data_t, data_n, data_p, data_d):
            vrad = datum_v[0]
            vtan = datum_v[1]
            dtime = datetime_from_list(time)
            if started_at <= dtime <= stopped_at:
                dkey = dtime.strftime(precision)
                x_hci = None
                y_hci = None
                if dkey in orbit_data:
                    x_hci = orbit_data[dkey][0]
                    y_hci = orbit_data[dkey][1]
                all_data[dkey] = (
                    dtime.strftime("%Y-%m-%dT%H:%M:%S+00:00"),
                    vrad, vtan, sqrt(vrad * vrad + vtan * vtan),
                    datum_b, datum_t, datum_n, datum_p, datum_d,
                    x_hci, y_hci
                )
        cdf_handle.close()

    return all_data


def generate_csv_contents(target_slug, started_at, stopped_at):
    target_config = get_target_config(target_slug)
    log.debug("Crunching CSV contents for '%s'..." % target_config['name'])
    si = StringIO.StringIO()
    cw = csv_writer(si)
    cw.writerow(PROPERTIES)

    all_data = get_data_for_target(target_config, started_at, stopped_at)

    log.debug("Writing and sorting CSV for '%s'..." % target_config['slug'])
    for dkey in sorted(all_data):
        cw.writerow(all_data[dkey])

    log.info("Generated CSV contents for '%s'." % target_config['slug'])
    return si.getvalue()


def generate_csv_file_if_needed(target_slug, started_at, stopped_at):
    filename = "%s_%s_%s.csv" % (target_slug,
                                 started_at.strftime(FILE_DATE_FMT),
                                 stopped_at.strftime(FILE_DATE_FMT))
    local_csv_file = get_path("../cache/%s" % filename)
    if not isfile(local_csv_file):
        log.info("Generating CSV '%s'..." % local_csv_file)
        try:
            with open(local_csv_file, mode="w+") as f:
                f.write(generate_csv_contents(target_slug,
                                              started_at=started_at,
                                              stopped_at=stopped_at))
            log.info("Generation of '%s' done." % filename)
        except Exception as e:
            if isfile(local_csv_file):
                log.warn("Removing failed CSV '%s'..." % local_csv_file)
                removefile(local_csv_file)
            abort(500, "Failed creating CSV '%s' : %s" % (filename, e))


def remove_files_created_before(date, in_directory):
    """
    Will throw if something horrible happens.
    Does not remove recursively (could be done with os.walk if needed).
    Does not remove directories either.
    :param date: datetime object
    :param in_directory: absolute path to directory
    :return:
    """
    import os
    import time

    secs = time.mktime(date.timetuple())

    if not os.path.isdir(in_directory):
        raise ValueError("No directory to clean at '%s'.")

    removed_files = []
    for file_name in os.listdir(in_directory):
        file_path = os.path.join(in_directory, file_name)
        if os.path.isfile(file_path):
            t = os.stat(file_path)
            if t.st_ctime < secs:
                os.remove(file_path)
                removed_files.append(file_path)

    return removed_files


def get_hit_counter():
    hit_count_path = get_path("../VISITS")

    if isfile(hit_count_path):
        hit_count = int(open(hit_count_path).read())
    else:
        hit_count = 1

    return hit_count


def increment_hit_counter():
    hit_count_path = get_path("../VISITS")

    if isfile(hit_count_path):
        hit_count = int(open(hit_count_path).read())
        hit_count += 1
    else:
        hit_count = 1

    hit_counter_file = open(hit_count_path, 'w')
    hit_counter_file.write(str(hit_count))
    hit_counter_file.close()

    return hit_count


tpl_global_vars['visits'] = get_hit_counter()


# ROUTING #####################################################################

@app.route('/favicon.ico')
def favicon():
    return send_from_directory(
        join(app.root_path, 'static', 'img'),
        'favicon.ico', mimetype='image/vnd.microsoft.icon'
    )


@app.route("/")
@app.route("/home.html")
@app.route("/index.html")
def home():
    increment_hit_counter()
    return render_view('home.html.jinja2', {
        'targets': config['targets'],
        'planets': [s for s in config['targets'] if s['type'] == 'planet'],
        'probes':  [s for s in config['targets'] if s['type'] == 'probe'],
        'comets':  [s for s in config['targets'] if s['type'] == 'comet'],
        'visits':  get_hit_counter(),
    })


@app.route("/<target>_<started_at>_<stopped_at>.csv")
def download_target_csv(target, started_at, stopped_at):
    """
    Grab data and orbit data for the specified `target`,
    rearrange it and return it as a CSV file.
    `started_at` and `stopped_at` should be UTC.
    """
    check_target_config(target)
    try:
        started_at = datetime.datetime.strptime(started_at, FILE_DATE_FMT)
    except:
        abort(400, "Invalid started_at parameter : '%s'." % started_at)
    try:
        stopped_at = datetime.datetime.strptime(stopped_at, FILE_DATE_FMT)
    except:
        abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at)

    filename = "%s_%s_%s.csv" % (target,
                                 started_at.strftime(FILE_DATE_FMT),
                                 stopped_at.strftime(FILE_DATE_FMT))
    local_csv_file = get_path("../cache/%s" % filename)
    generate_csv_file_if_needed(target, started_at, stopped_at)
    if not isfile(local_csv_file):
        abort(500, "Could not cache CSV file at '%s'." % local_csv_file)

    return send_from_directory(get_path("../cache/"), filename)


@app.route("/<targets>_<started_at>_<stopped_at>.tar.gz")
def download_targets_tarball(targets, started_at, stopped_at):
    """
    Grab data and orbit data for each of the specified `targets`,
    in their own CSV file, and make a tarball of them.
    `started_at` and `stopped_at` should be UTC strings.

    targets: string list of targets' slugs, separated by `-`.
    """
    separator = '-'
    targets = targets.split(separator)
    targets.sort()
    targets_configs = []
    for target in targets:
        if not target:
            abort(400, "Invalid targets format : `%s`." % targets)
        targets_configs.append(get_target_config(target))
    if 0 == len(targets_configs):
        abort(400, "No valid targets specified. What are you doing?")

    date_fmt = "%Y-%m-%dT%H:%M:%S"
    try:
        started_at = datetime.datetime.strptime(started_at, date_fmt)
    except:
        abort(400, "Invalid started_at parameter : '%s'." % started_at)
    try:
        stopped_at = datetime.datetime.strptime(stopped_at, date_fmt)
    except:
        abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at)
    sta = started_at.strftime(date_fmt)
    sto = stopped_at.strftime(date_fmt)

    gzip_filename = "%s_%s_%s.tar.gz" % (separator.join(targets), sta, sto)
    local_gzip_file = get_path("../cache/%s" % gzip_filename)

    if not isfile(local_gzip_file):
        log.debug("Creating the CSV files for the tarball...")
        for target_config in targets_configs:
            filename = "%s_%s_%s.csv" % (target_config['slug'], sta, sto)
            local_csv_file = get_path("../cache/%s" % filename)
            if not isfile(local_csv_file):
                with open(local_csv_file, mode="w+") as f:
                    f.write(generate_csv_contents(target_config['slug'],
                                                  started_at=started_at,
                                                  stopped_at=stopped_at))

        log.debug("Creating the tarball '%s'..." % local_gzip_file)
        with tarfile.open(local_gzip_file, "w:gz") as tar:
            for target_config in targets_configs:
                filename = "%s_%s_%s.csv" % (target_config['slug'], sta, sto)
                local_csv_file = get_path("../cache/%s" % filename)
                tar.add(local_csv_file, arcname=filename)

    if not isfile(local_gzip_file):
        abort(500, "No tarball to serve. Looked at '%s'." % local_gzip_file)

    return send_from_directory(get_path("../cache/"), gzip_filename)


@app.route("/<targets>_<params>_<started_at>_<stopped_at>.nc")
def download_targets_netcdf(targets, params, started_at, stopped_at):
    """
    Grab data and orbit data for the specified `target`,
    rearrange it and return it as a CSV file.
    `started_at` and `stopped_at` should be UTC.

    targets: string list of targets' slugs, separated by `-`.
    params: string list of targets' parameters, separated by `-`.
    """
    separator = '-'
    targets = targets.split(separator)
    targets.sort()
    targets_configs = []
    for target in targets:
        if not target:
            abort(400, "Invalid targets format : `%s`." % targets)
        targets_configs.append(get_target_config(target))
    if 0 == len(targets_configs):
        abort(400, "No valid targets specified. What are you doing?")
    params = params.split(separator)
    params.sort()
    if 0 == len(params):
        abort(400, "No valid parameters specified. What are you doing?")
    if not is_list_in_list(params, PARAMETERS.keys()):
        abort(400, "Some parameters are not recognized in '%s'." % str(params))

    date_fmt = "%Y-%m-%dT%H:%M:%S"
    try:
        started_at = datetime.datetime.strptime(started_at, date_fmt)
    except:
        abort(400, "Invalid started_at parameter : '%s'." % started_at)
    try:
        stopped_at = datetime.datetime.strptime(stopped_at, date_fmt)
    except:
        abort(400, "Invalid stopped_at parameter : '%s'." % stopped_at)
    sta = started_at.strftime(date_fmt)
    sto = stopped_at.strftime(date_fmt)

    nc_filename = "%s_%s_%s_%s.nc" % \
                  (separator.join(targets), separator.join(params), sta, sto)
    nc_path = get_path("../cache/%s" % nc_filename)

    if not isfile(nc_path):
        log.debug("Creating the NetCDF file '%s'..." % nc_filename)
        nc_handle = Dataset(nc_path, "w", format="NETCDF4")
        try:
            nc_handle.description = "TODO"  # todo
            nc_handle.history = "Created " + time.ctime(time.time())
            nc_handle.source = "Transplanet (CDDP)"
            available_params = list(PROPERTIES)
            for target in targets_configs:
                target_slug = target['slug']
                log.debug("Adding group '%s' to the NetCDF..." % target_slug)
                nc_group = nc_handle.createGroup(target_slug)
                data = get_data_for_target(target, started_at, stopped_at)
                dkeys = sorted(data)
                nc_handle.createDimension('dim_'+target_slug, len(dkeys))
                # todo: add time
                nc_vars = []
                indices = []
                for param in params:
                    indices.append(available_params.index(param))
                    nc_var = nc_group.createVariable(
                        param, 'f8', ('dim_'+target_slug,)
                    )
                    nc_var.units = PARAMETERS[param]['unit']
                    nc_vars.append(nc_var)
                for i, nc_var in enumerate(nc_vars):
                    index = indices[i]
                    values = []
                    for dkey in dkeys:
                        dval = data[dkey]
                        values.append(dval[index])
                    nc_var[:] = values
                # todo: add orbit x and y
        except Exception as e:
            raise e
        finally:
            nc_handle.close()

    if not isfile(nc_path):
        abort(500, "No NetCDF to serve. Looked at '%s'." % nc_path)

    return send_from_directory(get_path("../cache/"), nc_filename)


# API #########################################################################

@app.route("/cache/clear")
def cache_clear():
    """
    Removes all files from the cache that are older than roughly one month.
    """
    a_month_ago = datetime.datetime.now() - datetime.timedelta(days=32)
    cache_dir = get_path('../cache')
    removed_files = remove_files_created_before(a_month_ago, cache_dir)
    return "Cache cleared! Removed %d old file(s)." % len(removed_files)


# DEV TOOLS ###################################################################

# @app.route("/inspect")
# def analyze_cdf():
#     """
#     For debug purposes.
#     """
#     cdf_to_inspect = get_path("../res/dummy.nc")
#     cdf_to_inspect = get_path("../res/dummy_jupiter_coordinates.nc")
#
#     si = StringIO.StringIO()
#     cw = csv.DictWriter(si, fieldnames=['Name', 'Shape', 'Length'])
#     cw.writeheader()
#
#     # Time, StartTime, StopTime, V, B, N, T, Delta_angle, P_dyn, QualityFlag
#     cdf_handle = Dataset(cdf_to_inspect, "r", format="NETCDF4")
#     for variable in cdf_handle.variables:
#         v = cdf_handle.variables[variable]
#         cw.writerow({
#             'Name': variable,
#             'Shape': v.shape,
#             'Length': v.size,
#         })
#     cdf_handle.close()
#
#     return si.getvalue()


# MAIN ########################################################################

if __name__ == "__main__":
    # Debug mode on, as the production server does not use this.
    extra_files = [get_path('../config.yml')]
    app.run(debug=True, extra_files=extra_files)