{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ac074036-9b0f-4c95-817d-e407f2bebc7b",
   "metadata": {},
   "source": [
    "### We want to change the getdata api calls from amda url, to speasy library.\n",
    "\n",
    "#### First we take a look at how it was done, and what "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "011e9aa5-2047-431d-810f-28c37652fc77",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Logger HelioPropa (DEBUG)>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import sys\n",
    "import logging\n",
    "import matplotlib.pyplot as plt\n",
    "from datetime import datetime\n",
    "from speasy import amda\n",
    "import pandas as pd\n",
    "from pprint import pprint\n",
    "\n",
    "\n",
    "#\n",
    "# Heliopropa imports\n",
    "#\n",
    "\n",
    "# try to tweak the logging system first\n",
    "#\n",
    "os.environ['DEBUG'] = '0'\n",
    "\n",
    "sys.path.insert(0, os.path.abspath('..'))\n",
    "from web.run import get_target_config,\\\n",
    "                    get_data_for_target,\\\n",
    "                    FILE_DATE_FMT,\\\n",
    "                    generate_csv_contents,\\\n",
    "                    generate_csv_contents_spz,\\\n",
    "                    init_console_logger\n",
    "\n",
    "init_console_logger()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3df7573-9d30-4f08-a9d1-5040466387df",
   "metadata": {},
   "source": [
    "##### Configure dates targets and inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b38c421f-b879-4b39-a504-0667a9930aff",
   "metadata": {},
   "outputs": [],
   "source": [
    "# \n",
    "_target_slug = \"jupiter\"\n",
    "_input_slug = \"om\"\n",
    "_start_time =  datetime.strptime('20230101', '%Y%m%d')\n",
    "_stop_time =  datetime.strptime('20230401', '%Y%m%d')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56b4d003-fc08-4351-a68d-96abec9787da",
   "metadata": {},
   "source": [
    "##### Get data with old method (amda http requests)\n",
    "\n",
    "Returns dictionnary of tupples indexed by dates.\n",
    "Each tuple contains 10 values:\n",
    "\n",
    "\n",
    "time,vrad,vtan,vtot,btan,temp,pdyn,dens,atse,xhee,yhee"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f580de31-734f-47c2-a09e-c44cbe465a0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "target_config = get_target_config( _target_slug)\n",
    "all_data = get_data_for_target(target_config, _input_slug, _start_time, _stop_time )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "885d8f5e-6632-4f7c-9242-548ba475662c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-01-01T00 (418.201, 1.91232, 418.2053547600748, -1.28504, 21.042585, 0.189691, 0.649458, 73.5269, 0.19168941175309254, -4.946060186087171)\n",
      "2023-01-01T01 (427.381, 2.00023, 427.3856937825598, -1.42265, 23.80293, 0.221789, 0.727082, 73.6145, 0.18835391411799504, -4.946187857748381)\n",
      "2023-01-01T02 (426.966, 1.80149, 426.96981011542255, -1.36174, 24.530518, 0.214035, 0.70303, 73.7011, 0.18501833890839703, -4.946313278663125)\n",
      "2023-01-01T03 (425.964, 0.965016, 425.965080728456, -1.25969, 25.500776, 0.200314, 0.661068, 73.7923, 0.1816826873197642, -4.946436448571691)\n",
      "2023-01-01T04 (425.124, -0.2761, 425.1240811516092, -1.19995, 26.148018, 0.188949, 0.626032, 73.8916, 0.17834696051856475, -4.946557367267161)\n"
     ]
    }
   ],
   "source": [
    "# display first elements of the all_data retrieved this way\n",
    "small_data_keys = list(all_data.keys())[0:5]\n",
    "for k in small_data_keys:\n",
    "    # dont display first tuple element that is date\n",
    "    print(k, all_data[k][1:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ef11af95-c667-4e43-b1ea-cbb31caf3471",
   "metadata": {},
   "outputs": [],
   "source": [
    "# same with the calling method\n",
    "all_csv = generate_csv_contents(_target_slug, _input_slug, _start_time, _stop_time)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7c1a7e1e-dc0c-4b1a-b91e-847788594acd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['time,vrad,vtan,vtot,btan,temp,pdyn,dens,atse,xhee,yhee\\r', '2023-01-01T00:00:00+00:00,418.201,1.91232,418.2053547600748,-1.28504,21.042585,0.189691,0.649458,73.5269,0.19168941175309254,-4.946060186087171\\r', '2023-01-01T01:00:00+00:00,427.381,2.00023,427.3856937825598,-1.42265,23.80293,0.221789,0.727082,73.6145,0.18835391411799504,-4.946187857748381\\r', '2023-01-01T02:00:00+00:00,426.966,1.80149,426.96981011542255,-1.36174,24.530518,0.214035,0.70303,73.7011,0.18501833890839703,-4.946313278663125\\r', '2023-01-01T03:00:00+00:00,425.964,0.965016,425.965080728456,-1.25969,25.500776,0.200314,0.661068,73.7923,0.1816826873197642,-4.946436448571691\\r', '2023-01-01T04:00:00+00:00,425.124,-0.2761,425.1240811516092,-1.19995,26.148018,0.188949,0.626032,73.8916,0.17834696051856475,-4.946557367267161\\r', '2023-01-01T05:00:00+00:00,423.918,-0.98903,423.91915871425294,-1.18127,26.011208,0.182437,0.607898,73.9947,0.17501115965008188,-4.946676034598966\\r', '2023-01-01T06:00:00+00:00,422.428,-1.23374,422.42980186061686,-1.19154,25.33793,0.179517,0.602393,74.0981,0.17167528584658295,-4.9467924504753285\\r', '2023-01-01T07:00:00+00:00,420.961,-1.24559,420.96282867374407,-1.25493,23.961723,0.179086,0.605142,74.2005,0.1683393402357727,-4.946906614864552\\r', '2023-01-01T08:00:00+00:00,419.637,-1.0635,419.6383301427552,-1.37532,21.373276,0.182003,0.618888,74.3015,0.16500332394941092,-4.947018527795114\\r', '2023-01-01T09:00:00+00:00,418.925,-0.443971,418.9252245329708,-1.49301,18.073362,0.187551,0.639926,74.4034,0.1616672381319541,-4.947128189354561\\r', '2023-01-01T10:00:00+00:00,418.848,0.28136,418.848085527438,-1.55071,14.993965,0.197787,0.6751,74.5078,0.15833108394903567,-4.947235599687129\\r', '2023-01-01T11:00:00+00:00,419.019,0.834926,419.0198570772034,-1.56371,12.125517,0.215581,0.735231,74.612,0.1549948625956068,-4.947340758990131\\r', '2023-01-01T12:00:00+00:00,418.821,1.31839,418.82309063613,-1.55687,9.808448,0.238362,0.813691,74.7163,0.1516585753035288,-4.9474436675091304\\r', '2023-01-01T13:00:00+00:00,417.987,1.76892,417.9907295622715,-1.54465,8.2302065,0.259036,0.88779,74.8197,0.14832222334842712,-4.947544325531972\\r']\n"
     ]
    }
   ],
   "source": [
    "all_csv_aslist = all_csv.split('\\n')\n",
    "print(all_csv_aslist[0:15])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9748ab9f-3dc8-4b02-bf4f-a750a35864b5",
   "metadata": {},
   "source": [
    "#### Now rewrite that with speazy\n",
    "\n",
    "ie, generate a csv file with that new library.p\n",
    "That means getting parameters, rename the column as expected by the javascript grappher,\n",
    "and concatenate to get a single dataframe."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "05e8cec5-be4e-4a8a-b73c-469cc946ad05",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Orbit dataset : jupiter-orb-all\n",
      "Plsama dataset : tao-jup-sw\n"
     ]
    }
   ],
   "source": [
    "# First, get the dataset ids \n",
    "target_config = get_target_config(_target_slug)\n",
    "orbit_dataset_id = target_config['orbit']['models'][0]['slug']\n",
    "plasma_dataset_id = target_config['models'][_input_slug][0]['slug']\n",
    "orbit_dataset_id = orbit_dataset_id.replace('_', '-')\n",
    "plasma_dataset_id = plasma_dataset_id.replace('_', '-')\n",
    "\n",
    "print(f\"Orbit dataset : {orbit_dataset_id}\")\n",
    "print(f\"Plsama dataset : {plasma_dataset_id}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "af1b616b-d6e0-4ad5-baeb-cce3b9b96fd3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Then the dataset associated\n",
    "orbit_dataset = amda.get_dataset(orbit_dataset_id, _start_time, _stop_time)\n",
    "plasma_dataset = amda.get_dataset(plasma_dataset_id, _start_time, _stop_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e6ea5f1a-aeee-42c6-b031-a69466064333",
   "metadata": {},
   "outputs": [],
   "source": [
    "# vrad, vtan, vtot, btan, brad, pdyn, dens, atse, xhee, yhee\n",
    "\n",
    "# set a dict of amda parameters ids index by the expected column names in the csv\n",
    "\n",
    "params_dict =  {'dens': 'mars_sw_n',\n",
    "                'xy_v': 'mars_sw_v',\n",
    "                'temp': 'mars_sw_t',\n",
    "                'pdyn': 'mars_sw_pdyn',\n",
    "                'btan': 'mars_sw_b',\n",
    "                'brad': 'mars_sw_bx',\n",
    "                'atse': 'mars_sw_da',\n",
    "                'xy_hee': 'xyz_mars_hee'}\n",
    "\n",
    "list_df = []\n",
    "for _name, _id in params_dict.items():\n",
    "    _df = amda.get_data(_id, _start_time, _stop_time).to_dataframe()\n",
    "    if _name == 'xy_v':\n",
    "        _df = _df.rename(columns={_df.columns[0]:'vrad', _df.columns[1]:'vtan'})\n",
    "    elif _name == 'xy_hee':\n",
    "        _df = _df.drop(_df.columns[2], axis=1)\n",
    "        _df = _df.rename(columns={_df.columns[0]:'xhee', _df.columns[1]:'yhee'})\n",
    "    else:\n",
    "        _df = _df.rename(columns={_df.columns[0]: _name})\n",
    "\n",
    "    # _df = _df[~_df.index.duplicated()]\n",
    "\n",
    "    # resample to frequency, for later concatenation\n",
    "    _df = _df.resample('1H').mean()\n",
    "\n",
    "    # _df = _df.loc[_df.first_valid_index():_df.last_valid_index()]\n",
    "\n",
    "    list_df.append( _df)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8696bcec-5d54-411a-ad9f-8b161f00556b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                        vrad      vtan        vtot      btan      brad  \\\n",
      "2023-01-01 00:00:00  458.539  -8.99439  458.627205 -1.892408  0.490593   \n",
      "2023-01-01 01:00:00  539.666 -18.05740  539.968019 -2.591060  2.102028   \n",
      "2023-01-01 02:00:00  543.801 -16.74380  544.058712 -2.522670  2.062160   \n",
      "2023-01-01 03:00:00  547.864 -13.52640  548.030953 -2.502990  2.061292   \n",
      "2023-01-01 04:00:00  549.258  -4.20118  549.274067 -2.487440  2.053632   \n",
      "\n",
      "                         temp      pdyn      dens     atse      xhee      yhee  \n",
      "2023-01-01 00:00:00   6.42018  0.713528  2.027965  10.3768  1.528606 -0.330547  \n",
      "2023-01-01 01:00:00  10.00060  0.721630  1.482050  10.6671  1.528529 -0.331140  \n",
      "2023-01-01 02:00:00   9.38966  0.655797  1.326660  10.6880  1.528452 -0.331733  \n",
      "2023-01-01 03:00:00   8.68052  0.638148  1.272310  10.7095  1.528375 -0.332326  \n",
      "2023-01-01 04:00:00   7.80526  0.629647  1.249690  10.7316  1.528297 -0.332920  \n"
     ]
    }
   ],
   "source": [
    "from math import sqrt\n",
    "final_df = pd.concat(list_df, axis=1)\n",
    "# Is ther a vtot param ? else build it\n",
    "if 'vtot' not in final_df.columns:\n",
    "    final_df['vtot'] = final_df.apply(lambda x: sqrt(x.vtan*x.vtan+ x.vrad*x.vrad), axis=1)\n",
    "\n",
    "cols_ordered = ['vrad', 'vtan', 'vtot', 'btan', 'brad', 'temp', 'pdyn', 'dens', 'atse', 'xhee', 'yhee']\n",
    "final_df = final_df[cols_ordered]\n",
    "\n",
    "# and show\n",
    "pd.set_option('display.max_columns', None)\n",
    "print(final_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "eaec4f41-2524-4562-935a-42b4ce3fdae0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time\n"
     ]
    }
   ],
   "source": [
    "final_df.index.name = 'time'\n",
    "print(final_df.index.name)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bde656ba-98b9-4b03-93a3-c0121be49388",
   "metadata": {},
   "source": [
    "#### Use config to get the parameters dict and send to new method\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f92a9195-c863-48e4-ae62-2b3adcd66143",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_csv_content_spz(target_slug, input_slug, started_at, stopped_at):\n",
    "    target_config = get_target_config(target_slug)\n",
    "    params_dict = target_config['models'][input_slug][0]['parameters']\n",
    "    \n",
    "    list_df = []\n",
    "    for _name, _id in params_dict.items():\n",
    "        _df = amda.get_data(_id, _start_time, _stop_time).to_dataframe()\n",
    "        if _name == 'xy_v':\n",
    "            _df = _df.rename(columns={_df.columns[0]:'vrad', _df.columns[1]:'vtan'})\n",
    "        elif _name == 'xy_hee':\n",
    "            _df = _df.drop(_df.columns[2], axis=1)\n",
    "            _df = _df.rename(columns={_df.columns[0]:'xhee', _df.columns[1]:'yhee'})\n",
    "        else:\n",
    "            _df = _df.rename(columns={_df.columns[0]: _name})\n",
    "    \n",
    "        # _df = _df[~_df.index.duplicated()]\n",
    "    \n",
    "        # resample to frequency, for later concatenation\n",
    "        _df = _df.resample('1H').mean()\n",
    "    \n",
    "        # _df = _df.loc[_df.first_valid_index():_df.last_valid_index()]\n",
    "    \n",
    "        list_df.append( _df)\n",
    "    \n",
    "    from math import sqrt\n",
    "    final_df = pd.concat(list_df, axis=1)\n",
    "    # Is ther a vtot param ? else build it\n",
    "    if 'vtot' not in final_df.columns:\n",
    "        final_df['vtot'] = final_df.apply(lambda x: sqrt(x.vtan*x.vtan+ x.vrad*x.vrad), axis=1)\n",
    "        \n",
    "    \n",
    "    cols_ordered = ['vrad', 'vtan', 'vtot', 'btan', 'brad', 'temp', 'pdyn', 'dens', 'atse', 'xhee', 'yhee']\n",
    "    final_df = final_df[cols_ordered]\n",
    "    final_df.index.name = 'time'\n",
    "    return final_df.to_csv(date_format='%Y-%m-%dT%H:%M:%S', float_format=\"%.5f\",\n",
    "                      header=True,\n",
    "                      sep=\",\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "434bbf7b-d201-4d72-aac5-2a3cac6a3967",
   "metadata": {},
   "outputs": [],
   "source": [
    "ch = logging.StreamHandler()\n",
    "ch.setLevel(logging.DEBUG)\n",
    "formatter = logging.Formatter('%(levelname)s-%(message)s')\n",
    "ch.setFormatter(formatter)\n",
    "_hp_logger = logging.getLogger(\"HelioPropa\")\n",
    "_hp_logger.addHandler(ch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c995ca1c-dc2e-4c0b-95fa-aab394a8cf2c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO-Aggregating dataframes speazy parameters for 'om' to 'mars'\n",
      "DEBUG-Getting parameter id 'mars_sw_n' for 'dens'\n",
      "DEBUG-Getting parameter id 'mars_sw_v' for 'xy_v'\n",
      "DEBUG-Getting parameter id 'mars_sw_t' for 'temp'\n",
      "DEBUG-Getting parameter id 'mars_sw_pdyn' for 'pdyn'\n",
      "DEBUG-Getting parameter id 'mars_sw_b' for 'btan'\n",
      "DEBUG-Getting parameter id 'mars_sw_bx' for 'brad'\n",
      "DEBUG-Getting parameter id 'mars_sw_da' for 'atse'\n",
      "DEBUG-Getting parameter id 'xyz_mars_hee' for 'xy_hee'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time,vrad,vtan,vtot,btan,brad,temp,pdyn,dens,atse,xhee,yhee\n",
      "2023-01-01T00:00:00+00:00,458.53900,-8.99439,458.62721,-1.89241,0.49059,6.42018,0.71353,2.02797,10.37680,1.52861,-0.33055\n",
      "2023-01-01T01:00:00+00:00,539.66600,-18.05740,539.96802,-2.59106,2.10203,10.00060,0.72163,1.48205,10.66710,1.52853,-0.33114\n",
      "2023-01-01T02:00:00+00:00,543.80100,-16.74380,544.05871,-2.52267,2.06216,9.38966,0.65580,1.32666,10.68800,1.52845,-0.33173\n",
      "2023-01-01T03:00:00+00:00,547.86400,-13.52640,548.03095,-2.50299,2.06129,8.68052,0.63815,1.27231,10.70950,1.52837,-0.33233\n",
      "2023-01-01T04:00:00+00:00,549.25800,-4.20118,549.27407,-2.48744,2.05363,7.80526,0.62965,1.24969,10.73160,1.52830,-0.33292\n",
      "2023-01-01T05:00:00+00:00,548.67600,5.81362,548.70680,-2.46739,2.03485,6.73197,0.64252,1.27788,10.75300,1.52822,-0.33351\n",
      "2023-01-01T06:00:00+00:00,546.76300,8.89294,546.83532,-2.38355,1.95879,6.23209,0.62952,1.26060,10.77400,1.52814,-0.33411\n",
      "2023-01-01T07:00:00+00:00,548.76000,8.77561,548.83016,-2.44846,2.01942,6.44499,0.66720,1.32637,10.79620,1.52806,-0.33470\n",
      "2023-01-01T08:00:00+00:00,568.55900,5.35430,568.58421,-3.28875,2.81024,8.80241,1.03575,1.91845,10.81750,1.52798,-0.33529\n",
      "2023-01-01T09:00:00+00:00,575.19300,3.52090,575.20378,-3.97863,3.43930,9.53319,1.20380,2.17869,10.83690,1.52791,-0.33589\n",
      "2023-01-01T10:00:00+00:00,572.05100,3.21963,572.06006,-4.17327,3.58773,9.32707,1.12910,2.06602,10.85580,1.52783,-0.33648\n",
      "2023-01-01T11:00:00+00:00,570.28400,1.84143,570.28697,-4.29914,3.68440,9.75793,1.05895,1.94971,10.87520,1.52775,-0.33707\n",
      "2023-01-01T12:00:00+00:00,571.52000,-4.19166,571.53537,-4.30552,3.69775,11.64290,1.05794,1.93936,10.89480,1.52767,-0.33767\n",
      "2023-01-01T13:00:00+00:00,571.01300,-7.97585,571.06870,-4.19236,3.59725,13.51050,1.06096,1.94808,10.91470,1.52759,-0.33826\n"
     ]
    }
   ],
   "source": [
    "csv_content = generate_csv_contents_spz('mars', 'om', '20230101', '20230115')\n",
    "for _l in csv_content.split('\\n')[:15]:\n",
    "    print(_l)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}