CDPP / HELIOPROPA

Commit 89c56528070ccccdc5ad1bd97373dda7b3f6daa9

Authored by hitier
1 parent e4a36ad2
Exists in rhitier-dev

Rename heliopropa logger

Showing 2 changed files with 674 additions and 67 deletions   Show diff stats
Inline Side-by-side
notebooks/Understanding_Config_YAML.ipynb 0 → 100644
  Diff comments   View file @89c5652
... ... @@ -0,0 +1,607 @@
  1 +{
  2 + "cells": [
  3 + {
  4 + "cell_type": "markdown",
  5 + "id": "1f0ee132-a51a-4f2b-8f22-2eeea48b4e09",
  6 + "metadata": {},
  7 + "source": [
  8 + "#### How we load"
  9 + ]
  10 + },
  11 + {
  12 + "cell_type": "code",
  13 + "execution_count": 28,
  14 + "id": "b43839b2-f9b3-48f3-a4de-77464eb35af1",
  15 + "metadata": {},
  16 + "outputs": [
  17 + {
  18 + "name": "stdout",
  19 + "output_type": "stream",
  20 + "text": [
  21 + "dict_keys(['meta', 'header', 'authors', 'amda', 'defaults', 'layers', 'inputs', 'targets', 'toots'])\n",
  22 + "[{'default': False,\n",
  23 + " 'locked': False,\n",
  24 + " 'models': {'art': [{'parameters': {'atse': 'mercury_swrt_da',\n",
  25 + " 'brad': 'mercury_swrt_bx',\n",
  26 + " 'btan': 'mercury_swrt_b',\n",
  27 + " 'dens': 'mercury_swrt_n',\n",
  28 + " 'pdyn': 'mercury_swrt_pdyn',\n",
  29 + " 'temp': 'mercury_swrt_t',\n",
  30 + " 'xy_v': 'mercury_swrt_v'},\n",
  31 + " 'slug': 'tao-mercury-swrt'}],\n",
  32 + " 'dsc': [{'parameters': {'atse': 'mercury_dsc_da',\n",
  33 + " 'brad': 'mercury_dsc_bx',\n",
  34 + " 'btan': 'mercury_dsc_b',\n",
  35 + " 'dens': 'mercury_dsc_n',\n",
  36 + " 'pdyn': 'mercury_dsc_pdyn',\n",
  37 + " 'temp': 'mercury_dsc_t',\n",
  38 + " 'xy_v': 'mercury_dsc_v'},\n",
  39 + " 'slug': 'tao-mercury-dsc'}],\n",
  40 + " 'om': [{'parameters': {'atse': 'mercury_sw_da',\n",
  41 + " 'brad': 'mercury_sw_bx',\n",
  42 + " 'btan': 'mercury_sw_b',\n",
  43 + " 'dens': 'mercury_sw_n',\n",
  44 + " 'pdyn': 'mercury_sw_pdyn',\n",
  45 + " 'temp': 'mercury_sw_t',\n",
  46 + " 'xy_v': 'mercury_sw_v'},\n",
  47 + " 'slug': 'tao-mercury-sw'}],\n",
  48 + " 'sa': [{'parameters': {'atse': 'mercury_sta_da',\n",
  49 + " 'brad': 'mercury_sta_bx',\n",
  50 + " 'btan': 'mercury_sta_b',\n",
  51 + " 'dens': 'mercury_sta_n',\n",
  52 + " 'pdyn': 'mercury_sta_pdyn',\n",
  53 + " 'temp': 'mercury_sta_t',\n",
  54 + " 'xy_v': 'mercury_sta_v'},\n",
  55 + " 'slug': 'tao-mercury-sta'}],\n",
  56 + " 'sb': [{'parameters': {'atse': 'mercury_stb_da',\n",
  57 + " 'brad': 'mercury_stb_bx',\n",
  58 + " 'btan': 'mercury_stb_b',\n",
  59 + " 'dens': 'mercury_stb_n',\n",
  60 + " 'pdyn': 'mercury_stb_pdyn',\n",
  61 + " 'temp': 'mercury_stb_t',\n",
  62 + " 'xy_v': 'mercury_stb_v'},\n",
  63 + " 'slug': 'tao-mercury-stb'}],\n",
  64 + " 'solo': [{'slug': 'tao-mercury-solo'}]},\n",
  65 + " 'name': 'Mercury',\n",
  66 + " 'orbit': {'models': [{'parameters': {'xy_hee': 'xyz_mercury_hee'},\n",
  67 + " 'slug': 'mercury-orb-all'}],\n",
  68 + " 'semimajor': 0,\n",
  69 + " 'semiminor': 0},\n",
  70 + " 'slug': 'mercury',\n",
  71 + " 'title': 'Mercury',\n",
  72 + " 'type': 'planet'},\n",
  73 + " {'default': True,\n",
  74 + " 'locked': False,\n",
  75 + " 'models': {'art': [{'parameters': {'atse': 'venus_swrt_da',\n",
  76 + " 'brad': 'venus_swrt_bx',\n",
  77 + " 'btan': 'venus_swrt_b',\n",
  78 + " 'dens': 'venus_swrt_n',\n",
  79 + " 'pdyn': 'venus_swrt_pdyn',\n",
  80 + " 'temp': 'venus_swrt_t',\n",
  81 + " 'xy_v': 'venus_swrt_v'},\n",
  82 + " 'slug': 'tao-venus-swrt'}],\n",
  83 + " 'dsc': [{'parameters': {'atse': 'venus_dsc_da',\n",
  84 + " 'brad': 'venus_dsc_bx',\n",
  85 + " 'btan': 'venus_dsc_b',\n",
  86 + " 'dens': 'venus_dsc_n',\n",
  87 + " 'pdyn': 'venus_dsc_pdyn',\n",
  88 + " 'temp': 'venus_dsc_t',\n",
  89 + " 'xy_v': 'venus_dsc_v'},\n",
  90 + " 'slug': 'tao-venus-dsc'}],\n",
  91 + " 'om': [{'parameters': {'atse': 'venus_sw_da',\n",
  92 + " 'brad': 'venus_sw_bx',\n",
  93 + " 'btan': 'venus_sw_b',\n",
  94 + " 'dens': 'venus_sw_n',\n",
  95 + " 'pdyn': 'venus_sw_pdyn',\n",
  96 + " 'temp': 'venus_sw_t',\n",
  97 + " 'xy_v': 'venus_sw_v'},\n",
  98 + " 'slug': 'tao-venus-sw'}],\n",
  99 + " 'sa': [{'parameters': {'atse': 'venus_sta_da',\n",
  100 + " 'brad': 'venus_sta_bx',\n",
  101 + " 'btan': 'venus_sta_b',\n",
  102 + " 'dens': 'venus_sta_n',\n",
  103 + " 'pdyn': 'venus_sta_pdyn',\n",
  104 + " 'temp': 'venus_sta_t',\n",
  105 + " 'xy_v': 'venus_sta_v'},\n",
  106 + " 'slug': 'tao-venus-sta'}],\n",
  107 + " 'sb': [{'parameters': {'atse': 'venus_stb_da',\n",
  108 + " 'brad': 'venus_stb_bx',\n",
  109 + " 'btan': 'venus_stb_b',\n",
  110 + " 'dens': 'venus_stb_n',\n",
  111 + " 'pdyn': 'venus_stb_pdyn',\n",
  112 + " 'temp': 'venus_stb_t',\n",
  113 + " 'xy_v': 'venus_stb_v'},\n",
  114 + " 'slug': 'tao-venus-stb'}],\n",
  115 + " 'solo': [{'slug': 'tao-venus-solo'}]},\n",
  116 + " 'name': 'Venus',\n",
  117 + " 'orbit': {'models': [{'parameters': {'xy_hee': 'xyz_venus_hee'},\n",
  118 + " 'slug': 'venus-orb-all'}],\n",
  119 + " 'semimajor': 0.72333199,\n",
  120 + " 'semiminor': 0.7233154},\n",
  121 + " 'slug': 'venus',\n",
  122 + " 'title': 'Venus',\n",
  123 + " 'type': 'planet'},\n",
  124 + " {'default': True,\n",
  125 + " 'locked': False,\n",
  126 + " 'models': {'om': [{'parameters': {'pdyn': 'RamP'}, 'slug': 'omni_hour_all'},\n",
  127 + " {'parameters': {'dens': 'Dens',\n",
  128 + " 'temp': 'Temp',\n",
  129 + " 'vtot': 'Vel'},\n",
  130 + " 'slug': 'ace_swepam_real_1h'}],\n",
  131 + " 'sa': [{'parameters': {'pdyn': 'RamP'}, 'slug': 'omni_hour_all'},\n",
  132 + " {'parameters': {'dens': 'Dens',\n",
  133 + " 'temp': 'Temp',\n",
  134 + " 'vtot': 'Vel'},\n",
  135 + " 'slug': 'ace_swepam_real_1h'}],\n",
  136 + " 'sb': [{'parameters': {'pdyn': 'RamP'}, 'slug': 'omni_hour_all'},\n",
  137 + " {'parameters': {'dens': 'Dens',\n",
  138 + " 'temp': 'Temp',\n",
  139 + " 'vtot': 'Vel'},\n",
  140 + " 'slug': 'ace_swepam_real_1h'}]},\n",
  141 + " 'name': 'Earth',\n",
  142 + " 'orbit': {'models': []},\n",
  143 + " 'slug': 'earth',\n",
  144 + " 'title': 'Earth',\n",
  145 + " 'type': 'planet'},\n",
  146 + " {'default': False,\n",
  147 + " 'locked': False,\n",
  148 + " 'models': {'art': [{'parameters': {'atse': 'mars_swrt_da',\n",
  149 + " 'brad': 'mars_swrt_bx',\n",
  150 + " 'btan': 'mars_swrt_b',\n",
  151 + " 'dens': 'mars_swrt_n',\n",
  152 + " 'pdyn': 'mars_swrt_pdyn',\n",
  153 + " 'temp': 'mars_swrt_t',\n",
  154 + " 'xy_v': 'mars_swrt_v'},\n",
  155 + " 'slug': 'tao-mars-swrt'}],\n",
  156 + " 'dsc': [{'parameters': {'atse': 'mars_dsc_da',\n",
  157 + " 'brad': 'mars_dsc_bx',\n",
  158 + " 'btan': 'mars_dsc_b',\n",
  159 + " 'dens': 'mars_dsc_n',\n",
  160 + " 'pdyn': 'mars_dsc_pdyn',\n",
  161 + " 'temp': 'mars_dsc_t',\n",
  162 + " 'xy_v': 'mars_dsc_v'},\n",
  163 + " 'slug': 'tao-mars-dsc'}],\n",
  164 + " 'om': [{'parameters': {'atse': 'mars_sw_da',\n",
  165 + " 'brad': 'mars_sw_bx',\n",
  166 + " 'btan': 'mars_sw_b',\n",
  167 + " 'dens': 'mars_sw_n',\n",
  168 + " 'pdyn': 'mars_sw_pdyn',\n",
  169 + " 'temp': 'mars_sw_t',\n",
  170 + " 'xy_v': 'mars_sw_v'},\n",
  171 + " 'slug': 'tao-mars-sw'}],\n",
  172 + " 'sa': [{'parameters': {'atse': 'mars_sta_da',\n",
  173 + " 'brad': 'mars_sta_bx',\n",
  174 + " 'btan': 'mars_sta_b',\n",
  175 + " 'dens': 'mars_sta_n',\n",
  176 + " 'pdyn': 'mars_sta_pdyn',\n",
  177 + " 'temp': 'mars_sta_t',\n",
  178 + " 'xy_v': 'mars_sta_v'},\n",
  179 + " 'slug': 'tao-mars-sta'}],\n",
  180 + " 'sb': [{'parameters': {'atse': 'mars_stb_da',\n",
  181 + " 'brad': 'mars_stb_bx',\n",
  182 + " 'btan': 'mars_stb_b',\n",
  183 + " 'dens': 'mars_stb_n',\n",
  184 + " 'pdyn': 'mars_stb_pdyn',\n",
  185 + " 'temp': 'mars_stb_t',\n",
  186 + " 'xy_v': 'mars_stb_v'},\n",
  187 + " 'slug': 'tao-mars-stb'}],\n",
  188 + " 'solo': [{'slug': 'tao-mars-solo'}]},\n",
  189 + " 'name': 'Mars',\n",
  190 + " 'orbit': {'models': [{'parameters': {'xy_hee': 'xyz_mars_hee'},\n",
  191 + " 'slug': 'mars-orb-all'}],\n",
  192 + " 'semimajor': 1.52366231,\n",
  193 + " 'semiminor': 1.51700011},\n",
  194 + " 'slug': 'mars',\n",
  195 + " 'title': 'Mars',\n",
  196 + " 'type': 'planet'},\n",
  197 + " {'default': False,\n",
  198 + " 'locked': False,\n",
  199 + " 'models': {'art': [{'slug': 'tao_jup_swrt'}],\n",
  200 + " 'dsc': [{'slug': 'tao_jup_dsc'}],\n",
  201 + " 'om': [{'slug': 'tao_jup_sw'}],\n",
  202 + " 'sa': [{'slug': 'tao_jup_sta'}],\n",
  203 + " 'sb': [{'slug': 'tao_jup_stb'}],\n",
  204 + " 'solo': [{'slug': 'tao_jup_solo'}]},\n",
  205 + " 'name': 'Jupiter',\n",
  206 + " 'orbit': {'models': [{'slug': 'jupiter_orb_all'}],\n",
  207 + " 'semimajor': 5.45516759,\n",
  208 + " 'semiminor': 4.95155843},\n",
  209 + " 'slug': 'jupiter',\n",
  210 + " 'tap': {'target_name': 'Jupiter'},\n",
  211 + " 'title': 'Jupiter',\n",
  212 + " 'type': 'planet'},\n",
  213 + " {'default': False,\n",
  214 + " 'locked': False,\n",
  215 + " 'models': {'art': [{'slug': 'tao_sat_swrt'}],\n",
  216 + " 'dsc': [{'slug': 'tao_sat_dsc'}],\n",
  217 + " 'om': [{'slug': 'tao_sat_sw'}],\n",
  218 + " 'sa': [{'slug': 'tao_sat_sta'}],\n",
  219 + " 'sb': [{'slug': 'tao_sat_stb'}],\n",
  220 + " 'solo': [{'slug': 'tao_sat_solo'}]},\n",
  221 + " 'name': 'Saturn',\n",
  222 + " 'orbit': {'models': [{'slug': 'saturn_orb_all'}],\n",
  223 + " 'semimajor': 9.53707032,\n",
  224 + " 'semiminor': 9.5230773},\n",
  225 + " 'slug': 'saturn',\n",
  226 + " 'tap': {'target_name': 'Saturn'},\n",
  227 + " 'title': 'Saturn',\n",
  228 + " 'type': 'planet'},\n",
  229 + " {'default': False,\n",
  230 + " 'locked': False,\n",
  231 + " 'models': {'art': [{'slug': 'tao_ura_swrt'}],\n",
  232 + " 'dsc': [{'slug': 'tao_ura_dsc'}],\n",
  233 + " 'om': [{'slug': 'tao_ura_sw'}],\n",
  234 + " 'sa': [{'slug': 'tao_ura_sta'}],\n",
  235 + " 'sb': [{'slug': 'tao_ura_stb'}],\n",
  236 + " 'solo': [{'slug': 'tao_ura_solo'}]},\n",
  237 + " 'name': 'Uranus',\n",
  238 + " 'orbit': {'models': [{'slug': 'uranus_orb_all'}],\n",
  239 + " 'semimajor': 19.19,\n",
  240 + " 'semiminor': 19.17},\n",
  241 + " 'slug': 'uranus',\n",
  242 + " 'title': 'Uranus',\n",
  243 + " 'type': 'planet'},\n",
  244 + " {'default': False,\n",
  245 + " 'locked': False,\n",
  246 + " 'models': {'art': [{'slug': 'tao_nep_swrt'}],\n",
  247 + " 'dsc': [{'slug': 'tao_nep_dsc'}],\n",
  248 + " 'om': [{'slug': 'tao_nep_sw'}],\n",
  249 + " 'sa': [{'slug': 'tao_nep_sta'}],\n",
  250 + " 'sb': [{'slug': 'tao_nep_stb'}],\n",
  251 + " 'solo': [{'slug': 'tao_nep_solo'}]},\n",
  252 + " 'name': 'Neptune',\n",
  253 + " 'orbit': {'models': [{'slug': 'neptune_orb_all'}],\n",
  254 + " 'semimajor': 30.06896348,\n",
  255 + " 'semiminor': 30.06785516},\n",
  256 + " 'slug': 'neptune',\n",
  257 + " 'title': 'Neptune',\n",
  258 + " 'type': 'planet'},\n",
  259 + " {'default': False,\n",
  260 + " 'locked': False,\n",
  261 + " 'models': {'art': [{'slug': 'tao_mercury_swrt'}],\n",
  262 + " 'dsc': [{'slug': 'tao_mercury_dsc'}],\n",
  263 + " 'om': [{'slug': 'tao_mercury_sw'}],\n",
  264 + " 'sa': [{'slug': 'tao_mercury_sta'}],\n",
  265 + " 'sb': [{'slug': 'tao_mercury_stb'}],\n",
  266 + " 'solo': [{'slug': 'tao_mercury_solo'}]},\n",
  267 + " 'name': 'SoloColombo Source',\n",
  268 + " 'orbit': {'models': [{'slug': 'bepi_cruise_all'}, {'slug': 'earth_orb_all'}]},\n",
  269 + " 'slug': 'bepi_src',\n",
  270 + " 'title': 'SoloColombo Source',\n",
  271 + " 'type': 'source'},\n",
  272 + " {'default': False,\n",
  273 + " 'locked': False,\n",
  274 + " 'models': {'art': [{'slug': 'tao_bepi_swrt'}],\n",
  275 + " 'om': [{'slug': 'tao_bepi_sw'}],\n",
  276 + " 'sa': [{'slug': 'tao_bepi_sta'}]},\n",
  277 + " 'name': 'BepiColombo',\n",
  278 + " 'orbit': {'models': [{'slug': 'bepi_cruise_all'}]},\n",
  279 + " 'slug': 'bepi',\n",
  280 + " 'title': 'BepiColombo',\n",
  281 + " 'type': 'probe'},\n",
  282 + " {'default': False,\n",
  283 + " 'locked': False,\n",
  284 + " 'models': {'art': [{'slug': 'tao_mercury_swrt'}],\n",
  285 + " 'dsc': [{'slug': 'tao_mercury_dsc'}],\n",
  286 + " 'om': [{'slug': 'tao_mercury_sw'}],\n",
  287 + " 'sa': [{'slug': 'tao_mercury_sta'}],\n",
  288 + " 'sb': [{'slug': 'tao_mercury_stb'}],\n",
  289 + " 'solo': [{'slug': 'tao_mercury_solo'}]},\n",
  290 + " 'name': 'Solo Source',\n",
  291 + " 'orbit': {'models': [{'slug': 'so_orb_all'}, {'slug': 'earth_orb_all'}]},\n",
  292 + " 'slug': 'solo_src',\n",
  293 + " 'title': 'Solo Source',\n",
  294 + " 'type': 'source'},\n",
  295 + " {'default': False,\n",
  296 + " 'locked': False,\n",
  297 + " 'models': {'art': [{'slug': 'tao_so_swrt'}], 'om': [{'slug': 'tao_so_sw'}]},\n",
  298 + " 'name': 'SolarOrbiter',\n",
  299 + " 'orbit': {'models': [{'slug': 'so_orb_all'}]},\n",
  300 + " 'slug': 'solo',\n",
  301 + " 'title': 'SolarOrbiter',\n",
  302 + " 'type': 'probe'},\n",
  303 + " {'default': False,\n",
  304 + " 'locked': False,\n",
  305 + " 'models': {'art': [{'slug': 'tao_mercury_swrt'}],\n",
  306 + " 'dsc': [{'slug': 'tao_mercury_dsc'}],\n",
  307 + " 'om': [{'slug': 'tao_mercury_sw'}],\n",
  308 + " 'sa': [{'slug': 'tao_mercury_sta'}],\n",
  309 + " 'sb': [{'slug': 'tao_mercury_stb'}],\n",
  310 + " 'solo': [{'slug': 'tao_mercury_solo'}]},\n",
  311 + " 'name': 'Parker Source',\n",
  312 + " 'orbit': {'models': [{'slug': 'psp_orb_all'}, {'slug': 'earth_orb_all'}]},\n",
  313 + " 'slug': 'psp_src',\n",
  314 + " 'title': 'Parker Source',\n",
  315 + " 'type': 'source'},\n",
  316 + " {'default': False,\n",
  317 + " 'locked': False,\n",
  318 + " 'models': {'art': [{'slug': 'tao_psp_swrt'}],\n",
  319 + " 'om': [{'slug': 'tao_psp_sw'}],\n",
  320 + " 'sa': [{'slug': 'tao_psp_sta'}]},\n",
  321 + " 'name': 'Parker Solar Probe',\n",
  322 + " 'orbit': {'models': [{'slug': 'psp_orb_all'}]},\n",
  323 + " 'slug': 'psp',\n",
  324 + " 'title': 'Parker Solar Probe',\n",
  325 + " 'type': 'probe'},\n",
  326 + " {'default': False,\n",
  327 + " 'locked': False,\n",
  328 + " 'models': {'om': [{'slug': 'tao_ros_sw'}],\n",
  329 + " 'sa': [{'slug': 'tao_ros_sw'}],\n",
  330 + " 'sb': [{'slug': 'tao_ros_sw'}]},\n",
  331 + " 'name': 'Rosetta',\n",
  332 + " 'orbit': {'models': [{'slug': 'ros_orb_cruise',\n",
  333 + " 'stopped_at': '2014-08-02T00:00:00'},\n",
  334 + " {'slug': 'p67_orb_all',\n",
  335 + " 'started_at': '2014-08-02T00:00:00'}]},\n",
  336 + " 'slug': 'rosetta',\n",
  337 + " 'title': 'Rosetta',\n",
  338 + " 'type': 'probe'},\n",
  339 + " {'default': False,\n",
  340 + " 'locked': False,\n",
  341 + " 'models': {'om': [{'slug': 'tao_juno_sw'}],\n",
  342 + " 'sa': [{'slug': 'tao_juno_sw'}],\n",
  343 + " 'sb': [{'slug': 'tao_juno_sw'}]},\n",
  344 + " 'name': 'Juno',\n",
  345 + " 'orbit': {'models': [{'slug': 'juno_cruise_all',\n",
  346 + " 'stopped_at': '2016-07-05T03:53:00'},\n",
  347 + " {'slug': 'jupiter_orb_all',\n",
  348 + " 'started_at': '2016-07-05T03:53:00'}]},\n",
  349 + " 'slug': 'juno',\n",
  350 + " 'title': 'Juno',\n",
  351 + " 'type': 'probe'},\n",
  352 + " {'default': False,\n",
  353 + " 'locked': True,\n",
  354 + " 'models': {'om': [{'slug': 'tao_p67_sw'}],\n",
  355 + " 'sa': [{'slug': 'tao_p67_sw'}],\n",
  356 + " 'sb': [{'slug': 'tao_p67_sw'}]},\n",
  357 + " 'name': 'Churyumov-Gerasimenko',\n",
  358 + " 'orbit': {'models': [{'parameters': {'hee': 'XYZ_HEE'},\n",
  359 + " 'slug': 'p67_orb_all'}]},\n",
  360 + " 'slug': 'p67',\n",
  361 + " 'title': 'Churyumov-Gerasimenko (coming soon)',\n",
  362 + " 'type': 'comet'}]\n"
  363 + ]
  364 + }
  365 + ],
  366 + "source": [
  367 + "from yaml import load as yaml_load\n",
  368 + "from yaml import Loader\n",
  369 + "from pprint import pprint\n",
  370 + "\n",
  371 + "\n",
  372 + "config_file = '../my_cfg.yaml'\n",
  373 + "config_file = '../config.yml'\n",
  374 + "\n",
  375 + "with open(config_file, 'r', encoding='utf8') as config_file:\n",
  376 + " config = yaml_load(config_file.read(), Loader=Loader)\n",
  377 + "\n",
  378 + "\n",
  379 + "pprint(config.keys())\n",
  380 + "pprint(config['targets'])"
  381 + ]
  382 + },
  383 + {
  384 + "cell_type": "markdown",
  385 + "id": "48c3efcb-3f20-4ccc-b2ec-21bed3a14b9f",
  386 + "metadata": {},
  387 + "source": [
  388 + "#### How we read"
  389 + ]
  390 + },
  391 + {
  392 + "cell_type": "code",
  393 + "execution_count": null,
  394 + "id": "b02f3e74-0d95-4b3f-b696-a610a90f6e16",
  395 + "metadata": {},
  396 + "outputs": [],
  397 + "source": [
  398 + "import sys\n",
  399 + "import os\n",
  400 + "\n",
  401 + "sys.path.insert(0, os.path.abspath('..'))\n",
  402 + "from web.run import get_target_config"
  403 + ]
  404 + },
  405 + {
  406 + "cell_type": "markdown",
  407 + "id": "9aefa4ad-7600-4034-b90e-26be168f0850",
  408 + "metadata": {},
  409 + "source": [
  410 + "##### The 'targets' section of the config structure, is a list of dictionnaries"
  411 + ]
  412 + },
  413 + {
  414 + "cell_type": "code",
  415 + "execution_count": 36,
  416 + "id": "eb88fb52-705c-45e4-87ef-79fb37590991",
  417 + "metadata": {},
  418 + "outputs": [
  419 + {
  420 + "name": "stdout",
  421 + "output_type": "stream",
  422 + "text": [
  423 + " planet Mercury\n",
  424 + " planet Venus\n",
  425 + " planet Earth\n",
  426 + " planet Mars\n",
  427 + " planet Jupiter\n",
  428 + " planet Saturn\n",
  429 + " planet Uranus\n",
  430 + " planet Neptune\n",
  431 + " source SoloColombo Source\n",
  432 + " probe BepiColombo\n",
  433 + " source Solo Source\n",
  434 + " probe SolarOrbiter\n",
  435 + " source Parker Source\n",
  436 + " probe Parker Solar Probe\n",
  437 + " probe Rosetta\n",
  438 + " probe Juno\n",
  439 + " comet Churyumov-Gerasimenko\n"
  440 + ]
  441 + }
  442 + ],
  443 + "source": [
  444 + "for _t in config['targets']:\n",
  445 + " print(f\" {_t['type']:10} {_t['name']}\")"
  446 + ]
  447 + },
  448 + {
  449 + "cell_type": "markdown",
  450 + "id": "075f56b5-17f0-42b3-a982-a169c747345f",
  451 + "metadata": {},
  452 + "source": [
  453 + "##### Each planet or probe section contains a dict "
  454 + ]
  455 + },
  456 + {
  457 + "cell_type": "code",
  458 + "execution_count": 38,
  459 + "id": "49e2025d-cb8a-49a1-bb05-ecfef42b32ed",
  460 + "metadata": {},
  461 + "outputs": [
  462 + {
  463 + "name": "stdout",
  464 + "output_type": "stream",
  465 + "text": [
  466 + "dict_keys(['type', 'slug', 'name', 'title', 'orbit', 'models', 'locked', 'default'])\n"
  467 + ]
  468 + }
  469 + ],
  470 + "source": [
  471 + "\n",
  472 + "this_target_name = 'mars'\n",
  473 + "this_target_cfg = get_target_config(this_target_name)\n",
  474 + "pprint(this_target_cfg.keys())"
  475 + ]
  476 + },
  477 + {
  478 + "cell_type": "markdown",
  479 + "id": "58a14aa1-ba29-4b19-894b-99a1ef88654e",
  480 + "metadata": {},
  481 + "source": [
  482 + "##### in which we are mainly interested by 'orbit' and 'models' keys"
  483 + ]
  484 + },
  485 + {
  486 + "cell_type": "markdown",
  487 + "id": "1c1a6438-4f2c-4cbe-846f-3a36dd4922b7",
  488 + "metadata": {},
  489 + "source": [
  490 + "In the models section there is a dict indexed by the sources , each value being a list of dataset id with the amda ids of parameters."
  491 + ]
  492 + },
  493 + {
  494 + "cell_type": "code",
  495 + "execution_count": 41,
  496 + "id": "7c80f0cd-0a64-4b62-803f-7d813dd4b0cd",
  497 + "metadata": {},
  498 + "outputs": [
  499 + {
  500 + "name": "stdout",
  501 + "output_type": "stream",
  502 + "text": [
  503 + "models\n",
  504 + "------\n",
  505 + "{'art': [{'parameters': {'atse': 'mars_swrt_da',\n",
  506 + " 'brad': 'mars_swrt_bx',\n",
  507 + " 'btan': 'mars_swrt_b',\n",
  508 + " 'dens': 'mars_swrt_n',\n",
  509 + " 'pdyn': 'mars_swrt_pdyn',\n",
  510 + " 'temp': 'mars_swrt_t',\n",
  511 + " 'xy_v': 'mars_swrt_v'},\n",
  512 + " 'slug': 'tao-mars-swrt'}],\n",
  513 + " 'dsc': [{'parameters': {'atse': 'mars_dsc_da',\n",
  514 + " 'brad': 'mars_dsc_bx',\n",
  515 + " 'btan': 'mars_dsc_b',\n",
  516 + " 'dens': 'mars_dsc_n',\n",
  517 + " 'pdyn': 'mars_dsc_pdyn',\n",
  518 + " 'temp': 'mars_dsc_t',\n",
  519 + " 'xy_v': 'mars_dsc_v'},\n",
  520 + " 'slug': 'tao-mars-dsc'}],\n",
  521 + " 'om': [{'parameters': {'atse': 'mars_sw_da',\n",
  522 + " 'brad': 'mars_sw_bx',\n",
  523 + " 'btan': 'mars_sw_b',\n",
  524 + " 'dens': 'mars_sw_n',\n",
  525 + " 'pdyn': 'mars_sw_pdyn',\n",
  526 + " 'temp': 'mars_sw_t',\n",
  527 + " 'xy_v': 'mars_sw_v'},\n",
  528 + " 'slug': 'tao-mars-sw'}],\n",
  529 + " 'sa': [{'parameters': {'atse': 'mars_sta_da',\n",
  530 + " 'brad': 'mars_sta_bx',\n",
  531 + " 'btan': 'mars_sta_b',\n",
  532 + " 'dens': 'mars_sta_n',\n",
  533 + " 'pdyn': 'mars_sta_pdyn',\n",
  534 + " 'temp': 'mars_sta_t',\n",
  535 + " 'xy_v': 'mars_sta_v'},\n",
  536 + " 'slug': 'tao-mars-sta'}],\n",
  537 + " 'sb': [{'parameters': {'atse': 'mars_stb_da',\n",
  538 + " 'brad': 'mars_stb_bx',\n",
  539 + " 'btan': 'mars_stb_b',\n",
  540 + " 'dens': 'mars_stb_n',\n",
  541 + " 'pdyn': 'mars_stb_pdyn',\n",
  542 + " 'temp': 'mars_stb_t',\n",
  543 + " 'xy_v': 'mars_stb_v'},\n",
  544 + " 'slug': 'tao-mars-stb'}],\n",
  545 + " 'solo': [{'slug': 'tao-mars-solo'}]}\n"
  546 + ]
  547 + }
  548 + ],
  549 + "source": [
  550 + "print(\"models\\n------\")\n",
  551 + "pprint(this_target_cfg['models'])"
  552 + ]
  553 + },
  554 + {
  555 + "cell_type": "markdown",
  556 + "id": "ce2ff32d-8dde-45cf-8b47-11f0773da0f0",
  557 + "metadata": {},
  558 + "source": [
  559 + "In the orbit section there is a dict indexed by the sources , each value being a list of dataset id with the amda ids of parameters."
  560 + ]
  561 + },
  562 + {
  563 + "cell_type": "code",
  564 + "execution_count": 42,
  565 + "id": "2c8ca3c6-ef82-482b-b159-486b2177e35a",
  566 + "metadata": {},
  567 + "outputs": [
  568 + {
  569 + "name": "stdout",
  570 + "output_type": "stream",
  571 + "text": [
  572 + "orbit\n",
  573 + "-----\n",
  574 + "{'models': [{'parameters': {'xy_hee': 'xyz_mars_hee'}, 'slug': 'mars-orb-all'}],\n",
  575 + " 'semimajor': 1.52366231,\n",
  576 + " 'semiminor': 1.51700011}\n"
  577 + ]
  578 + }
  579 + ],
  580 + "source": [
  581 + "print(\"orbit\\n-----\")\n",
  582 + "pprint(this_target_cfg['orbit'])"
  583 + ]
  584 + }
  585 + ],
  586 + "metadata": {
  587 + "kernelspec": {
  588 + "display_name": "Python 3 (ipykernel)",
  589 + "language": "python",
  590 + "name": "python3"
  591 + },
  592 + "language_info": {
  593 + "codemirror_mode": {
  594 + "name": "ipython",
  595 + "version": 3
  596 + },
  597 + "file_extension": ".py",
  598 + "mimetype": "text/x-python",
  599 + "name": "python",
  600 + "nbconvert_exporter": "python",
  601 + "pygments_lexer": "ipython3",
  602 + "version": "3.10.12"
  603 + }
  604 + },
  605 + "nbformat": 4,
  606 + "nbformat_minor": 5
  607 +}
... ...
web/run.py
  Diff comments   View file @89c5652
... ... @@ -72,16 +72,16 @@ DEBUG = environ.get('DEBUG') == 'true'
72 72  
73 73 LOG_FILE = get_path('run.log')
74 74  
75   -log = logging.getLogger("HelioPropa")
  75 +hp_logger = logging.getLogger("HelioPropa")
76 76 if DEBUG:
77   - log.setLevel(logging.DEBUG)
  77 + hp_logger.setLevel(logging.DEBUG)
78 78 else:
79   - log.setLevel(logging.ERROR)
  79 + hp_logger.setLevel(logging.ERROR)
80 80 logHandler = logging.FileHandler(LOG_FILE)
81 81 logHandler.setFormatter(logging.Formatter(
82 82 "%(asctime)s - %(levelname)s - %(message)s"
83 83 ))
84   -log.addHandler(logHandler)
  84 +hp_logger.addHandler(logHandler)
85 85  
86 86 # HARDCODED CONFIGURATION #####################################################
87 87  
... ... @@ -176,13 +176,13 @@ environ['CDF_LIB'] = CDF_LIB
176 176 app = Flask(__name__, root_path=THIS_DIRECTORY)
177 177 app.debug = DEBUG
178 178 if app.debug:
179   - log.info("Starting Flask app IN DEBUG MODE...")
  179 + hp_logger.info("Starting Flask app IN DEBUG MODE...")
180 180 else:
181   - log.info("Starting Flask app...")
  181 + hp_logger.info("Starting Flask app...")
182 182  
183 183  
184 184 def handle_error(e):
185   - log.error(e)
  185 + hp_logger.error(e)
186 186 return str(e) # wish we could use the default error renderer here
187 187  
188 188  
... ... @@ -280,7 +280,7 @@ tpl_global_vars = {
280 280 # HELPERS #####################################################################
281 281  
282 282 def abort(code, message):
283   - log.error("Abort: " + message)
  283 + hp_logger.error("Abort: " + message)
284 284 abort_flask(code, message)
285 285  
286 286  
... ... @@ -343,7 +343,7 @@ def datetime_from_list(time_list):
343 343 try:
344 344 time_list = [str(i, 'UTF8') for i in time_list]
345 345 except Exception as e:
346   - log.error(e)
  346 + hp_logger.error(e)
347 347 # Day Of Year starts at 0, but for our datetime parser it starts at 1
348 348 doy = '{:03d}'.format(int(''.join(time_list[4:7])) + 1)
349 349 return datetime.datetime.strptime(
... ... @@ -561,7 +561,7 @@ def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at):
561 561 startTime=started_at.isoformat(),
562 562 stopTime=stopped_at.isoformat()
563 563 )
564   - log.info("Fetching remote gzip files list at '%s'." % url)
  564 + hp_logger.info("Fetching remote gzip files list at '%s'." % url)
565 565 retries = 0
566 566 success = False
567 567 errors = []
... ... @@ -581,21 +581,21 @@ def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at):
581 581 # raise Exception("API says it's out of time at '%s'." % url)
582 582 success = True
583 583 except Exception as e:
584   - log.warning("Failed (%d/3) '%s' : %s" % (retries + 1, url, e.message))
  584 + hp_logger.warning("Failed (%d/3) '%s' : %s" % (retries + 1, url, e.message))
585 585 remote_gzip_files = []
586 586 errors.append(e)
587 587 finally:
588 588 retries += 1
589 589 if not remote_gzip_files:
590   - log.error("Failed to retrieve data from AMDA.")
591   - log.error("Failed to fetch gzip files list for %s at '%s' : %s" %
592   - (orbiter, url, errors))
  590 + hp_logger.error("Failed to retrieve data from AMDA.")
  591 + hp_logger.error("Failed to fetch gzip files list for %s at '%s' : %s" %
  592 + (orbiter, url, errors))
593 593 abort(400, "Failed to fetch gzip files list for %s at '%s' : %s" %
594 594 (orbiter, url, errors))
595 595 else:
596 596 remote_gzip_files = list(set(remote_gzip_files))
597 597  
598   - log.debug("Fetched remote gzip files list : %s." % str(remote_gzip_files))
  598 + hp_logger.debug("Fetched remote gzip files list : %s." % str(remote_gzip_files))
599 599  
600 600 local_gzip_files = []
601 601 for remote_gzip_file in remote_gzip_files:
... ... @@ -609,16 +609,16 @@ def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at):
609 609 local_gzip_file = join(CACHE_DIR, get_local_filename(remote_gzip_file))
610 610 local_gzip_files.append(local_gzip_file)
611 611 if not isfile(local_gzip_file):
612   - log.debug("Retrieving '%s'..." % local_gzip_file)
  612 + hp_logger.debug("Retrieving '%s'..." % local_gzip_file)
613 613 urllib_request.urlretrieve(remote_gzip_file, local_gzip_file)
614   - log.debug("Retrieved '%s'." % local_gzip_file)
  614 + hp_logger.debug("Retrieved '%s'." % local_gzip_file)
615 615 else:
616   - log.debug("Found '%s' in the cache." % local_gzip_file)
  616 + hp_logger.debug("Found '%s' in the cache." % local_gzip_file)
617 617  
618 618 local_netc_files = []
619 619 for local_gzip_file in local_gzip_files:
620 620 local_netc_file = local_gzip_file[0:-3]
621   - log.debug("Unzipping '%s'..." % local_gzip_file)
  621 + hp_logger.debug("Unzipping '%s'..." % local_gzip_file)
622 622 success = True
623 623 try:
624 624 with gzip.open(local_gzip_file) as f:
... ... @@ -627,14 +627,14 @@ def retrieve_amda_netcdf(orbiter, what, started_at, stopped_at):
627 627 g.write(file_content)
628 628 except Exception as e:
629 629 success = False
630   - log.error("Cannot process gz file '%s' from '%s' : %s" %
631   - (local_gzip_file, url, e))
  630 + hp_logger.error("Cannot process gz file '%s' from '%s' : %s" %
  631 + (local_gzip_file, url, e))
632 632 # Sometimes, the downloaded gz is corrupted, and CRC checks fail.
633 633 # We want to delete the local gz file and try again next time.
634 634 removefile(local_gzip_file)
635 635 if success:
636 636 local_netc_files.append(local_netc_file)
637   - log.debug("Unzipped '%s'." % local_gzip_file)
  637 + hp_logger.debug("Unzipped '%s'." % local_gzip_file)
638 638  
639 639 return list(set(local_netc_files)) # remove possible dupes
640 640  
... ... @@ -689,7 +689,7 @@ def get_data_for_target(target_config, input_slug,
689 689 """
690 690 :return: dict whose keys are datetime as str, values tuples of data
691 691 """
692   - log.debug("Grabbing data for '%s'..." % target_config['slug'])
  692 + hp_logger.debug("Grabbing data for '%s'..." % target_config['slug'])
693 693  
694 694 try:
695 695 models = target_config['models'][input_slug]
... ... @@ -717,19 +717,19 @@ def get_data_for_target(target_config, input_slug,
717 717 target_config['slug'], orbit['slug'], s0, s1
718 718 )
719 719 for orbit_file in orbit_files:
720   - log.debug("%s: opening orbit NETCDF4 '%s'..." %
721   - (target_config['name'], orbit_file))
  720 + hp_logger.debug("%s: opening orbit NETCDF4 '%s'..." %
  721 + (target_config['name'], orbit_file))
722 722 cdf_handle = Dataset(orbit_file, "r", format="NETCDF4")
723 723 times = cdf_handle.variables['Time'] # YYYY DOY HH MM SS .ms
724 724 data_hee = _read_var(cdf_handle, nc_keys, 'hee', mandatory=True)
725 725  
726   - log.debug("%s: aggregating data from '%s'..." %
727   - (target_config['name'], orbit_file))
  726 + hp_logger.debug("%s: aggregating data from '%s'..." %
  727 + (target_config['name'], orbit_file))
728 728 for ltime, datum_hee in zip(times, data_hee):
729 729 try:
730 730 dtime = datetime_from_list(ltime)
731 731 except Exception:
732   - log.error("Failed to parse time from get__data_for_target %s." % ltime)
  732 + hp_logger.error("Failed to parse time from get__data_for_target %s." % ltime)
733 733 raise
734 734 # Keep only what's in the interval
735 735 if s0 <= dtime <= s1:
... ... @@ -749,12 +749,12 @@ def get_data_for_target(target_config, input_slug,
749 749 nc_keys.update(model['parameters'])
750 750  
751 751 if len(model_files) == 0:
752   - log.warning("No model data for '%s' '%s'."
753   - % (target_config['slug'], model['slug']))
  752 + hp_logger.warning("No model data for '%s' '%s'."
  753 + % (target_config['slug'], model['slug']))
754 754  
755 755 for model_file in model_files:
756   - log.debug("%s: opening model NETCDF4 '%s'..." %
757   - (target_config['name'], model_file))
  756 + hp_logger.debug("%s: opening model NETCDF4 '%s'..." %
  757 + (target_config['name'], model_file))
758 758 cdf_handle = Dataset(model_file, "r", format="NETCDF4")
759 759  
760 760 # log.debug(cdf_handle.variables.keys())
... ... @@ -782,15 +782,15 @@ def get_data_for_target(target_config, input_slug,
782 782 # FIXME
783 783 ignored_count = 0
784 784  
785   - log.debug("%s: aggregating data from '%s'..." %
786   - (target_config['name'], model_file))
  785 + hp_logger.debug("%s: aggregating data from '%s'..." %
  786 + (target_config['name'], model_file))
787 787 for ltime, datum_v, datum_b, datum_t, datum_n, datum_p, datum_a \
788 788 in zip(times, data_v, data_b, data_t, data_n, data_p, data_a):
789 789  
790 790 try:
791 791 dtime = datetime_from_list(ltime)
792 792 except Exception:
793   - log.error("Failed to parse time from %s." % ltime)
  793 + hp_logger.error("Failed to parse time from %s." % ltime)
794 794 raise
795 795  
796 796 if not (s0 <= dtime <= s1):
... ... @@ -857,8 +857,8 @@ def get_data_for_target(target_config, input_slug,
857 857  
858 858 # Improve this loop so as to remove this stinky debug log
859 859 if ignored_count > 0:
860   - log.debug(" Ignored %d datum(s) during ~\"drizzling\"."
861   - % ignored_count)
  860 + hp_logger.debug(" Ignored %d datum(s) during ~\"drizzling\"."
  861 + % ignored_count)
862 862  
863 863 cdf_handle.close()
864 864  
... ... @@ -867,7 +867,7 @@ def get_data_for_target(target_config, input_slug,
867 867  
868 868 def generate_csv_contents(target_slug, input_slug, started_at, stopped_at):
869 869 target_config = get_target_config(target_slug)
870   - log.debug("Crunching CSV contents for '%s'..." % target_config['name'])
  870 + hp_logger.debug("Crunching CSV contents for '%s'..." % target_config['name'])
871 871 si = StringIO()
872 872 cw = csv_writer(si)
873 873 cw.writerow(PROPERTIES)
... ... @@ -877,11 +877,11 @@ def generate_csv_contents(target_slug, input_slug, started_at, stopped_at):
877 877 started_at=started_at, stopped_at=stopped_at
878 878 )
879 879  
880   - log.debug("Writing and sorting CSV for '%s'..." % target_config['slug'])
  880 + hp_logger.debug("Writing and sorting CSV for '%s'..." % target_config['slug'])
881 881 for dkey in sorted(all_data):
882 882 cw.writerow(all_data[dkey])
883 883  
884   - log.info("Generated CSV contents for '%s'." % target_config['slug'])
  884 + hp_logger.info("Generated CSV contents for '%s'." % target_config['slug'])
885 885 return si.getvalue()
886 886  
887 887  
... ... @@ -891,10 +891,10 @@ def generate_csv_contents_spz(target_slug, input_slug, started_at, stopped_at):
891 891 orbit_dict = target_config['orbit']['models'][0]['parameters']
892 892 parameters_dict = {**plasma_dict, **orbit_dict}
893 893  
894   - log.info(f"Aggregating dataframes speazy parameters for '{input_slug}' to '{target_slug}'" )
  894 + hp_logger.info(f"Aggregating dataframes speazy parameters for '{input_slug}' to '{target_slug}'")
895 895 list_df = []
896 896 for _name, _id in parameters_dict.items():
897   - log.debug(f"Getting parameter id '{_id}' for '{_name}'")
  897 + hp_logger.debug(f"Getting parameter id '{_id}' for '{_name}'")
898 898 _df = amda.get_data(_id, started_at, stopped_at).to_dataframe()
899 899 if _name == 'xy_v':
900 900 _df = _df.rename(columns={_df.columns[0]: 'vrad', _df.columns[1]: 'vtan'})
... ... @@ -952,23 +952,23 @@ def generate_csv_file_if_needed(target_slug, input_slug,
952 952 csv_generator = generate_csv_contents
953 953  
954 954 if generate:
955   - log.info("Generating CSV '%s'..." % local_csv_file)
  955 + hp_logger.info("Generating CSV '%s'..." % local_csv_file)
956 956 try:
957 957 with open(local_csv_file, mode="w+") as f:
958 958 f.write(csv_generator(
959 959 target_slug=target_slug, input_slug=input_slug,
960 960 started_at=started_at, stopped_at=stopped_at
961 961 ))
962   - log.info("Generation of '%s' done." % filename)
  962 + hp_logger.info("Generation of '%s' done." % filename)
963 963 except Exception as e:
964 964 from sys import exc_info
965 965 from traceback import extract_tb
966 966 exc_type, exc_value, exc_traceback = exc_info()
967   - log.error(e)
  967 + hp_logger.error(e)
968 968 for trace in extract_tb(exc_traceback):
969   - log.error(trace)
  969 + hp_logger.error(trace)
970 970 if isfile(local_csv_file):
971   - log.warning("Removing failed CSV '%s'..." % local_csv_file)
  971 + hp_logger.warning("Removing failed CSV '%s'..." % local_csv_file)
972 972 removefile(local_csv_file)
973 973 abort(500, "Failed creating CSV '%s' : %s" % (filename, e))
974 974  
... ... @@ -1074,7 +1074,7 @@ def get_catalog_layers(input_slug, target_slug, started_at, stopped_at):
1074 1074 try:
1075 1075 index = _data['columns'].index(_key)
1076 1076 except ValueError:
1077   - log.error("Key %s not found in columns of %s" % (_key, _data))
  1077 + hp_logger.error("Key %s not found in columns of %s" % (_key, _data))
1078 1078 raise
1079 1079 return index
1080 1080  
... ... @@ -1106,7 +1106,7 @@ def get_catalog_layers(input_slug, target_slug, started_at, stopped_at):
1106 1106 with open(get_path("../data/catalog/%s" % cl_datum['file'])) as f:
1107 1107 json_data = json.load(f)
1108 1108 if 'start' not in cl_datum:
1109   - log.error("Invalid configuration: 'start' is missing.")
  1109 + hp_logger.error("Invalid configuration: 'start' is missing.")
1110 1110 continue # skip this
1111 1111 if 'format' not in cl_datum:
1112 1112 cl_datum['format'] = CME_DATE_FMT
... ... @@ -1189,12 +1189,12 @@ def update_spacepy():
1189 1189 Importing pycdf will fail if the toolbox is not up to date.
1190 1190 """
1191 1191 try:
1192   - log.info("Updating spacepy's toolbox…")
  1192 + hp_logger.info("Updating spacepy's toolbox…")
1193 1193 import spacepy.toolbox
1194 1194  
1195 1195 spacepy.toolbox.update()
1196 1196 except Exception as e:
1197   - log.error("Failed to update spacepy : %s." % e)
  1197 + hp_logger.error("Failed to update spacepy : %s." % e)
1198 1198  
1199 1199  
1200 1200 tpl_global_vars['visits'] = get_hit_counter()
... ... @@ -1330,7 +1330,7 @@ def download_targets_tarball(targets, inp, started_at, stopped_at):
1330 1330 local_gzip_file = join(CACHE_DIR, gzip_filename)
1331 1331  
1332 1332 if not isfile(local_gzip_file):
1333   - log.debug("Creating the CSV files for the tarball...")
  1333 + hp_logger.debug("Creating the CSV files for the tarball...")
1334 1334 for target_config in targets_configs:
1335 1335 filename = "%s_%s_%s_%s.csv" % (
1336 1336 target_config['slug'], input_slug, sta, sto
... ... @@ -1345,7 +1345,7 @@ def download_targets_tarball(targets, inp, started_at, stopped_at):
1345 1345 input_slug=input_slug
1346 1346 ))
1347 1347  
1348   - log.debug("Creating the tarball '%s'..." % local_gzip_file)
  1348 + hp_logger.debug("Creating the tarball '%s'..." % local_gzip_file)
1349 1349 with tarfile.open(local_gzip_file, "w:gz") as tar:
1350 1350 for target_config in targets_configs:
1351 1351 filename = "%s_%s_%s_%s.csv" % (
... ... @@ -1410,7 +1410,7 @@ def download_targets_netcdf(targets, inp, params, started_at, stopped_at):
1410 1410 nc_path = join(CACHE_DIR, nc_filename)
1411 1411  
1412 1412 if not isfile(nc_path):
1413   - log.debug("Creating the NetCDF file '%s'..." % nc_filename)
  1413 + hp_logger.debug("Creating the NetCDF file '%s'..." % nc_filename)
1414 1414 nc_handle = Dataset(nc_path, "w", format="NETCDF4")
1415 1415 try:
1416 1416 nc_handle.description = "Model and orbit data for targets" # todo
... ... @@ -1419,7 +1419,7 @@ def download_targets_netcdf(targets, inp, params, started_at, stopped_at):
1419 1419 available_params = list(PROPERTIES)
1420 1420 for target in targets_configs:
1421 1421 target_slug = target['slug']
1422   - log.debug("Adding group '%s' to the NetCDF..." % target_slug)
  1422 + hp_logger.debug("Adding group '%s' to the NetCDF..." % target_slug)
1423 1423 nc_group = nc_handle.createGroup(target_slug)
1424 1424 data = get_data_for_target(
1425 1425 target_config=target, input_slug=input_slug,
... ... @@ -1473,10 +1473,10 @@ def download_targets_netcdf(targets, inp, params, started_at, stopped_at):
1473 1473 values_y.append(dval[index_y])
1474 1474 nc_x[:] = values_x
1475 1475 nc_y[:] = values_y
1476   - log.debug("Writing NetCDF '%s'..." % nc_filename)
  1476 + hp_logger.debug("Writing NetCDF '%s'..." % nc_filename)
1477 1477  
1478 1478 except Exception:
1479   - log.error("Failed to generate NetCDF '%s'." % nc_filename)
  1479 + hp_logger.error("Failed to generate NetCDF '%s'." % nc_filename)
1480 1480 raise
1481 1481 finally:
1482 1482 nc_handle.close()
... ... @@ -1529,7 +1529,7 @@ def download_targets_cdf(targets, inp, started_at, stopped_at):
1529 1529 cdf_path = join(CACHE_DIR, cdf_filename)
1530 1530  
1531 1531 if not isfile(cdf_path):
1532   - log.debug("Creating the CDF file '%s'..." % cdf_filename)
  1532 + hp_logger.debug("Creating the CDF file '%s'..." % cdf_filename)
1533 1533 try:
1534 1534 from spacepy import pycdf
1535 1535 except ImportError:
... ... @@ -1539,10 +1539,10 @@ def download_targets_cdf(targets, inp, started_at, stopped_at):
1539 1539 try:
1540 1540 from spacepy import pycdf
1541 1541 except ImportError as e:
1542   - log.error("Failed to import pycdf from spacepy : %s" % e)
  1542 + hp_logger.error("Failed to import pycdf from spacepy : %s" % e)
1543 1543 raise
1544 1544 except Exception as e:
1545   - log.error("Failed to import pycdf from spacepy : %s" % e)
  1545 + hp_logger.error("Failed to import pycdf from spacepy : %s" % e)
1546 1546 raise
1547 1547  
1548 1548 try:
... ... @@ -1618,18 +1618,18 @@ def download_targets_cdf(targets, inp, started_at, stopped_at):
1618 1618 else:
1619 1619 values_xhee.append(0)
1620 1620 values_yhee.append(0)
1621   - log.warning("Orbit data for %s has NaNs." % target_slug)
  1621 + hp_logger.warning("Orbit data for %s has NaNs." % target_slug)
1622 1622 cdf_handle[kx] = values_xhee
1623 1623 cdf_handle[ky] = values_yhee
1624 1624 cdf_handle[kx].attrs['UNITS'] = 'Au'
1625 1625 cdf_handle[ky].attrs['UNITS'] = 'Au'
1626 1626  
1627   - log.debug("Writing CDF '%s'..." % cdf_filename)
  1627 + hp_logger.debug("Writing CDF '%s'..." % cdf_filename)
1628 1628 cdf_handle.close()
1629   - log.debug("Wrote CDF '%s'." % cdf_filename)
  1629 + hp_logger.debug("Wrote CDF '%s'." % cdf_filename)
1630 1630  
1631 1631 except Exception as e:
1632   - log.error("Failed to generate CDF '%s'." % cdf_filename)
  1632 + hp_logger.error("Failed to generate CDF '%s'." % cdf_filename)
1633 1633 if isfile(cdf_path):
1634 1634 removefile(cdf_path)
1635 1635 raise
... ... @@ -1643,7 +1643,7 @@ def download_targets_cdf(targets, inp, started_at, stopped_at):
1643 1643 @app.route("/<target>_auroral_catalog.csv")
1644 1644 def download_auroral_catalog_csv(target):
1645 1645 tc = validate_tap_target_config(target)
1646   - log.debug("Requesting auroral emissions CSV for %s..." % tc['name'])
  1646 + hp_logger.debug("Requesting auroral emissions CSV for %s..." % tc['name'])
1647 1647  
1648 1648 filename = "%s_auroral_catalog.csv" % (target)
1649 1649 local_csv_file = join(CACHE_DIR, filename)
... ... @@ -1670,10 +1670,10 @@ def download_auroral_catalog_csv(target):
1670 1670 # 'time_min', 'time_max', 'thumbnail_url', 'external_link'
1671 1671 # cw.writerow(head)
1672 1672  
1673   - log.debug("Writing auroral emissions CSV for %s..." % tc['name'])
  1673 + hp_logger.debug("Writing auroral emissions CSV for %s..." % tc['name'])
1674 1674 cw.writerows(emissions)
1675 1675  
1676   - log.info("Generated auroral emissions CSV contents for %s." % tc['name'])
  1676 + hp_logger.info("Generated auroral emissions CSV contents for %s." % tc['name'])
1677 1677 return si.getvalue()
1678 1678  
1679 1679 # if not isfile(local_csv_file):
... ...