CDPP / DDServer

Commit 4e0002b301a016ce19d71935f84b3ea32917c6a4

Authored by Benjamin Renard
2 parents b47b5d6a 6544052e
Exists in master and in 3 other branches WIP, evol_9042, ubuntu

Merge branch 'master' into evol_9042

Showing 4 changed files with 404 additions and 104 deletions   Show diff stats
Inline Side-by-side
src/DDADMIN/MANAGER/UserManagerClass.php
  Diff comments   View file @4e0002b
... ... @@ -205,7 +205,7 @@ class UserManagerClass
205 205 $msg .= "at http://amda.cdpp.eu \r\n\r\n";
206 206 $msg .= "Please contact us in case of any problems or questions.\r\n\r\n";
207 207 $msg .= "For acknowledgment and data use policy please follow the rules of the road available at http://amda.cdpp.eu/help/policy.html.\r\n\r\n";
208   -$msg .= "We also invite you to connect to our data archive (https://cdpp-archive.cnes.fr) and use our other tools for 3D visualization of spacecraft and science data (3DView, http://3dview.cdpp.eu), for solar wind propagation (Propagation Tool, http://propagationtool.cdpp.eu and Heliopropa, http://heliopropa.cdpp.eu), or for coordinate and time transformation (TREPS, http://treps.cdpp.eu)..\r\n\r\n";
  208 +$msg .= "We also invite you to connect to our data archive (https://cdpp-archive.cnes.fr) and use our other tools for 3D visualization of spacecraft and science data (3DView, http://3dview.cdpp.eu), for solar wind propagation (Propagation Tool, http://propagationtool.cdpp.eu and Heliopropa, http://heliopropa.cdpp.eu), or for coordinate and time transformations (TREPS, http://treps.cdpp.eu).\r\n\r\n";
209 209 $msg .= "Feedback on CDPP tools is always welcome and greatly appreciated (amda at irap.omp.eu)!\r\n\r\n";
210 210 $msg .= "Best regards,\r\n\r\n";
211 211 $msg .= "CDPP-AMDA Team";
... ...
src/REMOTEDATA/CDAWEB.php
  Diff comments   View file @4e0002b
... ... @@ -6,7 +6,9 @@
6 6 */
7 7 class CDAWEB extends RemoteDataCenterClass
8 8 {
9   - private $ch, $res, $dataViewUR;
  9 + private $dataViewURL = NULL;
  10 +
  11 + private $ch;
10 12 private $obsGroupsIds;
11 13 private $spase_res, $insXML, $xp = null;
12 14  
... ... @@ -19,7 +21,7 @@ class CDAWEB extends RemoteDataCenterClass
19 21 "Magnetic%20Fields%20(space)", "Particles%20(space)", "Plasma%20and%20Solar%20Wind",
20 22 "Radio%20and%20Plasma%20Waves%20(space)"]; //, "Ephemeris/Attitude/Ancillary" ];
21 23  
22   - private $excludeIns = ["GIFWALK", "Unknown"], $excludeGroup = ["THEMIS", "ARTEMIS", "MMS", "Apollo"]; // "NOAA", "GPS", "TSS-1R", "IBEX"];
  24 + private $excludeIns = ["GIFWALK", "Unknown"], $excludeGroup = ["THEMIS", "ARTEMIS", "Apollo"]; // "NOAA", "GPS", "TSS-1R", "IBEX"];
23 25  
24 26 // not in "https://heliophysicsdata.sci.gsfc.nasa.gov/queries/CDAWeb_SPASE.xql" List;
25 27 // FOR INFO : Excluded automatically
... ... @@ -66,7 +68,7 @@ class CDAWEB extends RemoteDataCenterClass
66 68 {
67 69 $this->ch = curl_init();
68 70 curl_setopt($this->ch, CURLOPT_RETURNTRANSFER, 1);
69   - curl_setopt($this->ch, CURLOPT_TIMEOUT, 60);
  71 + curl_setopt($this->ch, CURLOPT_TIMEOUT, 600);
70 72 // Add proxy definition
71 73 $PROXY_HOST=getenv('PROXY_HOST');
72 74 $PROXY_USERPWD=getenv('PROXY_USERPWD');
... ... @@ -85,35 +87,21 @@ class CDAWEB extends RemoteDataCenterClass
85 87  
86 88 private function initStreamContext()
87 89 {
  90 + $context = array(
  91 + 'http' => array(
  92 + 'timeout' => 600,
  93 + ),
  94 + );
88 95 $PROXY_HOST=getenv('PROXY_HOST');
89 96 $PROXY_USERPWD=getenv('PROXY_USERPWD');
90 97 if (!empty($PROXY_HOST)) {
91   - $context = array(
92   - 'http' => array(
93   - 'proxy' => "tcp://$PROXY_HOST",
94   - 'request_fulluri' => true,
95   - ),
96   - );
  98 + $context['http']['proxy'] = "tcp://$PROXY_HOST";
  99 + $context['http']['request_fulluri'] = TRUE;
97 100 if (!empty($PROXY_USERPWD)) {
98 101 $context['http']['header'] = "Proxy-Authorization: Basic ".base64_encode($PROXY_USERPWD);
99 102 }
100   - stream_context_set_default($context);
101   - }
102   - }
103   -
104   - protected function setDataViewURL()
105   - {
106   - curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$restUrl."/dataviews");
107   -
108   - $this->res = new DomDocument();
109   - $this->res->loadXML(curl_exec($this->ch));
110   -
111   - if ($this->res->getElementsByTagName('EndpointAddress')->length == 0) {
112   - error_log("Problem connect to ".CDAWebConfigClass::$restUrl."/dataviews", 3, err);
113   - exit("Problem connect to ".CDAWebConfigClass::$restUrl."/dataviews".PHP_EOL);
114 103 }
115   -
116   - $this->dataViewURL = $this->res->getElementsByTagName('EndpointAddress')->item(0)->nodeValue;
  104 + stream_context_set_default($context);
117 105 }
118 106  
119 107 /**
... ... @@ -123,24 +111,46 @@ class CDAWEB extends RemoteDataCenterClass
123 111 protected function getRemoteTree()
124 112 {
125 113 $this->openConnection();
126   -
127   - $this->setDataViewURL();
128 114 $this->obsGroupsIds = array();
129   -
130   - curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets");
131 115  
132   - $this->res->loadXML(curl_exec($this->ch));
133   - // $this->res->save("datasets.xml");
134   - // $this->res->load("datasets.xml");
135   - $datasets = $this->res->getElementsByTagName("DatasetDescription");
  116 + $dom = NULL;
  117 + $reuse_cache_file = FALSE;
  118 + if (file_exists($this->location."/datasets.xml")) {
  119 + if (time() - filemtime($this->location."/datasets.xml") < 86400) {
  120 + $dom = new DOMDocument();
  121 + if ($dom->load($this->location."/datasets.xml")) {
  122 + echo "Re-use cache file ".$this->location."/datasets.xml".PHP_EOL;
  123 + $reuse_cache_file = TRUE;
  124 + }
  125 + }
  126 + }
  127 +
  128 + if (!$reuse_cache_file) {
  129 + $dom = $this->loadFromCDAWebWS("datasets", array());
  130 + if (!$dom) {
  131 + error_log('Cannot retrieve CDAWeb datasets list');
  132 + return;
  133 + }
  134 + $dom->save($this->location."/datasets.xml");
  135 + }
  136 +
  137 + $datasets = $dom->getElementsByTagName("DatasetDescription");
136 138  
137 139 echo "All CDAWeb datasets : ".$datasets->length.PHP_EOL;
138 140  
139 141 foreach ($datasets as $ds) {
140   -
141   - $insType = $ds->getElementsByTagName("InstrumentType")->item(0)->nodeValue;
  142 +
  143 + $datasetId = $ds->getElementsByTagName("Id")->item(0)->nodeValue;
  144 + $insTypeNodes = $ds->getElementsByTagName("InstrumentType");
  145 + $validInstrumentType = FALSE;
  146 + foreach ($insTypeNodes as $insTypeNode) {
  147 + if (in_array ($this->html_encode($insTypeNode->nodeValue), $this->validInstrumentTypes )) {
  148 + $validInstrumentType = TRUE;
  149 + break;
  150 + }
  151 + }
142 152  
143   - if (in_array ($this->html_encode($insType), $this->validInstrumentTypes )) {
  153 + if ($validInstrumentType) {
144 154  
145 155 $id = $ds->getElementsByTagName("Id")->item(0)->nodeValue;
146 156  
... ... @@ -161,7 +171,7 @@ class CDAWEB extends RemoteDataCenterClass
161 171 if ( $obsGroupId == "UNKNOWN" ) {
162 172 $obsGroupId = $obsId;
163 173 }
164   -
  174 +
165 175 if ( $obsGroupId != "UNKNOWN" ) {
166 176 $this->obsGroupsIds[$obsGroupId][$obsId][$insId][] = $id;
167 177 }
... ... @@ -195,14 +205,16 @@ class CDAWEB extends RemoteDataCenterClass
195 205 {
196 206 if ( !array_key_exists($dsId, $this->CDAWEB ))
197 207 return null; // no description in SpaseRegistry => we do not add this dataset
198   -
  208 +
199 209 $dsNode = $this->domAmda->createElement('dataset');
  210 +
200 211  
201   - curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json"));
202   - curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets?idPattern=".$dsId);
  212 + $obj = $this->loadFromCDAWebWS("datasets", array("idPattern" => $dsId), TRUE);
  213 + if (!$obj) {
  214 + return NULL;
  215 + }
203 216  
204   - $obj = json_decode(curl_exec($this->ch));
205   - $dataSet = $obj->DatasetDescription;
  217 + $dataSet = $obj->DatasetDescription;
206 218  
207 219 $dsNode->setAttribute("xml:id",$this->baseID.":".$dsId);
208 220 $dsNode->setAttribute("name", $dsId);
... ... @@ -238,15 +250,20 @@ class CDAWEB extends RemoteDataCenterClass
238 250 }
239 251  
240 252 }
241   - else
  253 + else if (floatval($sampling) < 0.001) {
  254 + echo " Sampling too small : ".$dsId.PHP_EOL;
  255 + return NULL;
  256 + }
  257 + else {
242 258 $this->updateDDServerXml("MinSampling",$sampling);
243   -
  259 + }
  260 +
244 261 $dsNode->setAttribute('spaseUrl',$this->CDAWEB[$dsId]);
245 262 // $dsNode->setAttribute('masterCdf',$this->existsMasterCdf($dsId));
246 263 $dsNode->setAttribute("desc", "$label; $startTime - $endTime");
247 264 $dsNode->setAttribute("dataStart", $startTime);
248 265 $dsNode->setAttribute("dataStop", $endTime);
249   -
  266 +
250 267 $parameterNodes = $this->createParameterNodes($dsId);
251 268 foreach ($parameterNodes as $parameterNode)
252 269 {
... ... @@ -258,10 +275,9 @@ class CDAWEB extends RemoteDataCenterClass
258 275  
259 276 protected function createParameterNodes($dsId)
260 277 {
261   - curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json"));
262   - curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets/".$dsId."/variables");
263   -
264   - $obj = json_decode(curl_exec($this->ch));
  278 + $obj = $this->loadFromCDAWebWS("datasets/".$dsId."/variables", array(), TRUE);
  279 + if (!$obj)
  280 + return array();
265 281 $parameters = $obj->VariableDescription;
266 282  
267 283 $paramNodes = array();
... ... @@ -280,15 +296,11 @@ class CDAWEB extends RemoteDataCenterClass
280 296  
281 297 protected function getDatasetSpaseDescription($dsID)
282 298 {
283   - curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/xml"));
284   -
285 299 if ( !array_key_exists($dsID, $this->CDAWEB ))
286 300 return -1; // no description in SpaseRegistry
287   -
288   - curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$spaseResolver."id=".$this->CDAWEB[$dsID]);
289   -
290   - if (!$this->spase_res->loadXML(curl_exec($this->ch))) return -100;
291   -
  301 +
  302 + if (!$this->loadSpaseResource($this->CDAWEB[$dsID], $this->spase_res)) return -100;
  303 +
292 304 $messages = $this->spase_res->getElementsByTagName('Message');
293 305  
294 306 if ($messages->length > 0)
... ... @@ -327,27 +339,35 @@ class CDAWEB extends RemoteDataCenterClass
327 339 */
328 340 protected function getAllSpaseDatasets()
329 341 {
  342 + $this->initStreamContext();
  343 +
330 344 require_once "simple_html_dom.php";
331 345  
332   - if (file_exists($this->location."/NumericalData.html"))
333   - rename($this->location."/NumericalData.html",$this->location."/NumericalData.html.bak");
  346 + if (file_exists($this->location."/NumericalData.html"))
  347 + rename($this->location."/NumericalData.html",$this->location."/NumericalData.html.bak");
  348 +
334 349  
335 350 // if cannot reach CDAWEB Spase Registry use an old file
336   - if (!copy(CDAWebConfigClass::$spaseRegistry, $this->location."/NumericalData.html"))
  351 + if (!copy(CDAWebConfigClass::$spaseRegistry, $this->location."/NumericalData.html")) {
  352 + error_log('ERROR');
337 353 copy($this->location."/NumericalData.html.bak", $this->location."/NumericalData.html");
  354 + }
  355 +
  356 + $doc = new DOMDocument('1.0', 'utf-8');
  357 + if (!$doc->loadHTMLFile($this->location.'/NumericalData.html')) {
  358 + error_log("Cannot load ".$this->location.'/NumericalData.html');
  359 + }
  360 +
  361 + $xpath = new DOMXPath($doc);
  362 + $ids = $xpath->query("//td[@class='Spase.URL.ProductID']");
338 363  
339   - $html = file_get_html($this->location.'/NumericalData.html');
340   - $ids = $html->find('td[class="Spase.URL.ProductID"]');
341   -
342 364 foreach ($ids as $id)
343   - {
344   - if ($id->next_sibling()) {
345   - $ref = $id->next_sibling()->find('a');
346   - $key = $id->find('a');
347   - if (strlen($ref[0]->innertext) > 25)
348   - $this->CDAWEB[$key[0]->innertext] = $ref[0]->innertext; // if SPASE description exists
349   -
350   - }
  365 + {
  366 + $key = $id->nodeValue;
  367 + $spase = $id->nextSibling->nodeValue;
  368 + if (!empty($key) && (strlen($spase) > 25)) {
  369 + $this->CDAWEB[$key] = $spase;
  370 + }
351 371 }
352 372  
353 373 error_log("SPASE-defined CDAWEB datasets from ".CDAWebConfigClass::$spaseRegistry." : ".count($this->CDAWEB).PHP_EOL,3,log);
... ... @@ -415,23 +435,21 @@ class CDAWEB extends RemoteDataCenterClass
415 435 {
416 436 $this->initStreamContext();
417 437 $this->openConnection();
418   - $this->setDataViewURL();
419 438  
420   - curl_setopt($this->ch, CURLOPT_URL,$this->dataViewURL."/datasets/$ds/orig_data/$start,$stop/");
421   -
422   - $res = new DomDocument("1.0");
423   -
424   - $res->loadXML(curl_exec($this->ch));
425   -
426   - if ($res->getElementsByTagName("html")->length > 0)
427   - {
  439 + $res = $this->loadFromCDAWebWS("datasets/$ds/orig_data/$start,$stop/", array());
  440 + if (!$res) {
  441 + return array();
  442 + }
  443 +
  444 + if ($res->getElementsByTagName("html")->length > 0)
  445 + {
428 446 error_log("ERROR no response for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
429 447 error_log($res->saveXML(),3,err);
430 448 $this->closeConnection();
431 449 return array();
432   - }
  450 + }
433 451  
434   - $fileNames = $res->getElementsByTagName("Name");
  452 + $fileNames = $res->getElementsByTagName("Name");
435 453  
436 454 $nc_prefix = strlen($ds) > RemoteDataCenterClass::$MAX_VI_NAME_LENGTH ?
437 455 substr(strtolower($ds),0,RemoteDataCenterClass::$MAX_VI_NAME_LENGTH - 1): strtolower($ds);
... ... @@ -521,8 +539,8 @@ class CDAWEB extends RemoteDataCenterClass
521 539 foreach ($this->obsGroupsIds as $groupId => $obss)
522 540 {
523 541 if (in_array($groupId, $this->excludeGroup)) continue;
524   - echo $groupId.PHP_EOL;
525 542 $obsNodes = array();
  543 + echo $groupId.PHP_EOL;
526 544 foreach ($obss as $obs => $inss)
527 545 {
528 546 // ! observatory OMNI is taken for observatoriesGroup OMNI only !
... ... @@ -543,7 +561,7 @@ class CDAWEB extends RemoteDataCenterClass
543 561 if (!$this->domAmda->getElementById($dsId)){
544 562 $this->initDDServerXml($ds,$ins,$obs);
545 563 $dsNode = $this->createDatasetNode($ds);
546   -
  564 +
547 565 if ($dsNode){
548 566 $this->saveDDServerXml();
549 567 $dsNodes[] = $dsNode;
... ... @@ -555,16 +573,16 @@ class CDAWEB extends RemoteDataCenterClass
555 573 {
556 574 // last $spase_res : instrument should be the same
557 575 $insSpaseId = $this->getInstrumentSpase();
558   -
  576 +
559 577 if (!$insSpaseId || is_array($insSpaseId))
560 578 $insSpaseId = $ins;
561   -
  579 +
562 580 $obsSpaseId = strtolower($this->getObservatorySpase());
563 581 if (!$obsSpaseId )
564 582 $obsSpaseId = strtolower($obs);
565   -
  583 +
566 584 $insId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId).":".$this->param2dd($insSpaseId);
567   -
  585 +
568 586 if (!($insNode = $this->domAmda->getElementById($insId))){
569 587 $insNode = $this->createInstrumentNode($insSpaseId, $obsSpaseId, $groupId);
570 588 }
... ... @@ -576,7 +594,7 @@ class CDAWEB extends RemoteDataCenterClass
576 594 $insNodes[] = $insNode;
577 595 }
578 596 } // foreach ($inss as $ins => $dss)
579   -
  597 +
580 598 if (!empty($insNodes))
581 599 {
582 600 $obsId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId);
... ... @@ -591,6 +609,7 @@ class CDAWEB extends RemoteDataCenterClass
591 609 $obsNodes[] = $obsNode;
592 610 }
593 611 } // foreach ($obss as $obs => $inss)
  612 +
594 613  
595 614 if (!empty($obsNodes))
596 615 {
... ... @@ -617,7 +636,6 @@ class CDAWEB extends RemoteDataCenterClass
617 636 }
618 637 }
619 638  
620   -
621 639 $this->closeConnection();
622 640 }
623 641  
... ... @@ -634,18 +652,17 @@ class CDAWEB extends RemoteDataCenterClass
634 652 protected function getInstrumentSpase()
635 653 {
636 654 $this->insXML = new DomDocument("1.0");
637   -
  655 +
638 656 if ($this->spase_res){
639 657 $instrument = $this->spase_res->getElementsByTagName('InstrumentID');
640   -
  658 +
641 659 if ($instrument->length > 0) {
642 660 if ($instrument->length == 1) {
643   - curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$spaseResolver."id=".$instrument->item(0)->nodeValue);
644   - $this->insXML->loadXML(curl_exec($this->ch));
645   -
  661 + $this->loadSpaseResource($instrument->item(0)->nodeValue, $this->insXML);
  662 +
646 663 return $this->getIdFromSpase($instrument->item(0)->nodeValue);
647   - }
648   - else {
  664 + }
  665 + else {
649 666 // $insIds = "";
650 667 $insIds = array();
651 668 for ($i = 0; $i < $instrument->length; $i++ ) {
... ... @@ -655,8 +672,7 @@ class CDAWEB extends RemoteDataCenterClass
655 672 $insIds[] = $this->getIdFromSpase($instrument->item($i)->nodeValue);
656 673 }
657 674  
658   - curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$spaseResolver."id=".$instrument->item($i)->nodeValue);
659   - $this->insXML->loadXML(curl_exec($this->ch));
  675 + $this->loadSpaseResource($instrument->item($instrument->length - 1)->nodeValue, $this->insXML);
660 676 return $insIds;
661 677 }
662 678 //InstrumentType
... ... @@ -688,6 +704,74 @@ class CDAWEB extends RemoteDataCenterClass
688 704  
689 705 protected function makeArgumentsList(){}
690 706  
  707 +
  708 + private function loadSpaseResource($resourceID, &$dom) {
  709 + if (empty($resourceID)) {
  710 + return FALSE;
  711 + }
  712 + $url = str_replace('spase://', 'https://hpde.io/', $resourceID) . '.xml';
  713 + curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/xml"));
  714 + curl_setopt($this->ch, CURLOPT_URL, $url);
  715 + if (!$dom->loadXML(curl_exec($this->ch))) {
  716 + error_log('Cannot load Spase Resource: '.$resourceID);
  717 + return FALSE;
  718 + }
  719 + return TRUE;
  720 + }
  721 +
  722 + private function loadFromCDAWebWS($api, $params, $asJSON = FALSE) {
  723 + if (!$this->dataViewURL) {
  724 + curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$restUrl."/dataviews");
  725 + curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/xml"));
  726 +
  727 + $res = new DomDocument();
  728 + if (!$res->loadXML(curl_exec($this->ch))) {
  729 + return FALSE;
  730 + }
  731 +
  732 + if ($res->getElementsByTagName('EndpointAddress')->length == 0) {
  733 + error_log("Problem connect to ".CDAWebConfigClass::$restUrl."/dataviews", 3, err);
  734 + return FALSE;
  735 + }
  736 +
  737 + $this->dataViewURL = $res->getElementsByTagName('EndpointAddress')->item(0)->nodeValue;
  738 + }
  739 + $url = $this->dataViewURL . '/' . $api;
  740 + if (!empty($params)) {
  741 + $params_str = "";
  742 + foreach ($params as $key => $value) {
  743 + if (!empty($params_str)) {
  744 + $params_str .= "&";
  745 + }
  746 + $params_str .= $key."=".urlencode($value);
  747 + }
  748 + $url .= '?'.$params_str;
  749 + }
  750 +
  751 + if ($asJSON) {
  752 + curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json"));
  753 + }
  754 + else {
  755 + curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/xml"));
  756 + }
  757 + curl_setopt($this->ch, CURLOPT_URL, $url);
  758 +
  759 + $res = curl_exec($this->ch);
  760 + if (!$res) {
  761 + return FALSE;
  762 + }
  763 +
  764 + if ($asJSON) {
  765 + return json_decode($res);
  766 + }
  767 +
  768 + $dom = new DOMDocument();
  769 + if (!$dom->loadXML(curl_exec($this->ch))) {
  770 + return FALSE;
  771 + }
  772 +
  773 + return $dom;
  774 + }
691 775 }
692 776 ?>
693 777  
... ...
src/REMOTEDATA/CDAWebConfigClass.php
  Diff comments   View file @4e0002b
... ... @@ -13,11 +13,7 @@ class CDAWebConfigClass
13 13 // Master CDFs URL
14 14 public static $masterUrl = "https://cdaweb.gsfc.nasa.gov/pub/software/cdawlib/0MASTERS/";
15 15 // SPASE CDAWeb dataset descriptionlist
16   - public static $spaseRegistry = "https://heliophysicsdata.sci.gsfc.nasa.gov/queries/CDAWeb_SPASE.xql";
17   - // https://cdaweb.gsfc.nasa.gov/registry/hdp/NumericalData.xql : limited list
18   - // http://spase-group.org/registry/explorer/ SMWG
19   - // Service to get SPASE description by SPASE ID (from $spaseRegistry)
20   - public static $spaseResolver = "http://www.spase-group.org/registry/resolver?";
  16 + public static $spaseRegistry = "https://heliophysicsdata.gsfc.nasa.gov/queries/CDAWeb_SPASE.html";
21 17 public static $format = "CDF";
22 18 }
23 19 ?>
... ...
tests/checkDDBase.py 0 → 100644
  Diff comments   View file @4e0002b
... ... @@ -0,0 +1,220 @@
  1 +import os
  2 +import sys
  3 +import logging
  4 +from lxml import etree
  5 +import netCDF4 as nc
  6 +import numpy as np
  7 +import gzip
  8 +import pickle
  9 +from datetime import datetime
  10 +
  11 +vi_to_exclude = [
  12 + 'juno_ephem_orb1',
  13 + 'juno_fgm_orbfull',
  14 + 'juno_fgm_orb1',
  15 + 'juno_fgm_orb60',
  16 + 'juno_jedi_i090',
  17 + 'juno_jedi_i180',
  18 + 'juno_jedi_i270',
  19 + 'juno_jedi_e270',
  20 + 'juno_jedi_e180',
  21 + 'juno_jedi_e090',
  22 + 'juno_fgm_cruise60',
  23 + 'ros_magob_1s',
  24 +]
  25 +
  26 +def datetime_to_ddtime(date_time):
  27 + return "%04d%03d%02d%02d%02d%03d" % (date_time.year, int(date_time.timetuple().tm_yday)-1, date_time.hour, date_time.minute, date_time.second, date_time.microsecond/1000.)
  28 +
  29 +
  30 +def is_sorted(l):
  31 + return all(a <= b for a, b in zip(l, l[1:]))
  32 +
  33 +
  34 +def check_vi(e, cachedir):
  35 + name = e.find("NAME").text
  36 + logging.info('========== {} =========='.format(name))
  37 + base = e.find("NAME").attrib['base']
  38 + if base != 'LOCAL':
  39 + return True
  40 + location = e.find("LOCATION").text
  41 + times = e.find("TIMES").text
  42 + info = e.find("INFO").text
  43 + cache = e.find("CACHE").text
  44 +
  45 + if name == 'iball_acc_all':
  46 + # specific VI used to manage users
  47 + return True
  48 +
  49 + cachefile = os.path.join(cachedir, name)
  50 + if os.path.isfile(cachefile):
  51 + with open(cachefile, 'rb') as handle:
  52 + cache_check = pickle.load(handle)
  53 + else:
  54 + cache_check = {}
  55 +
  56 + if not os.path.isdir(location):
  57 + logging.error('{} not exists'.format(location))
  58 + return False
  59 +
  60 + times_path = os.path.join(location, times)
  61 + info_path = os.path.join(location, info)
  62 + cache_path = os.path.join(location, cache)
  63 +
  64 + for f in [times_path, info_path, cache_path]:
  65 + if not os.path.isfile(f):
  66 + logging.error('{} not exists'.format(f))
  67 + return False
  68 +
  69 + ds = nc.Dataset(times_path)
  70 +
  71 + for v in ['StartTime', 'StopTime', 'FileName']:
  72 + if v not in ds.variables:
  73 + logging.error('Missing {} variable in times file'.format(v))
  74 + return False
  75 +
  76 + start_times = []
  77 + for st in np.array(ds.variables["StartTime"]):
  78 + try:
  79 + st_str = "".join([k.decode("UTF-8") for k in st])
  80 + start_times.append(int(st_str))
  81 + except:
  82 + logging.error('Cannot parse StartTime in times file')
  83 + return False
  84 +
  85 + stop_times = []
  86 + for et in np.array(ds.variables["StopTime"]):
  87 + try:
  88 + et_str = "".join([k.decode("UTF-8") for k in et])
  89 + stop_times.append(int(et_str))
  90 + except:
  91 + logging.error('Cannot parse StopTime in times file')
  92 + return False
  93 +
  94 + files_names = []
  95 + for fn in np.array(ds.variables["FileName"]):
  96 + try:
  97 + fn_str = "".join([k.decode("UTF-8") for k in fn])
  98 + files_names.append(fn_str)
  99 + except:
  100 + logging.error('Cannot parse FileName in times file')
  101 + return False
  102 +
  103 + if len(start_times) != len(stop_times) or len(start_times) != len(files_names):
  104 + logging.error('Incoherence between variables size in times file')
  105 + return False
  106 +
  107 + if len(start_times) == 0:
  108 + logging.warning('Dataset is empty')
  109 + return True
  110 +
  111 + prev = None
  112 + for d in start_times:
  113 + if prev:
  114 + if d < prev:
  115 + logging.warning("Previous start time is higher {}".format(d))
  116 + prev = d
  117 +
  118 + prev = None
  119 + for d in stop_times:
  120 + if prev:
  121 + if d < prev:
  122 + logging.warning("Previous stop time is higher {}".format(d))
  123 + prev = d
  124 +
  125 + for i in range(len(start_times)):
  126 + if int(start_times[i]) > int(stop_times[i]):
  127 + logging.warning("Start time is higher than Stop time {} - {}".format(start_times[i], stop_times[i]))
  128 +
  129 + if os.path.isfile(os.path.join(location,'LOCK')):
  130 + logging.warning("LOCK file detected")
  131 +
  132 + for i in range(len(files_names)):
  133 + f = files_names[i]
  134 + start = start_times[i]
  135 + stop = stop_times[i]
  136 +
  137 + gzipped_f = os.path.join(location, f) + ".gz"
  138 +
  139 + if f in cache_check:
  140 + if cache_check[f]['status'] and os.path.getmtime(gzipped_f) == cache_check[f]['mtime']:
  141 + continue
  142 +
  143 + logging.info(f)
  144 +
  145 + if not os.path.isfile(gzipped_f):
  146 + logging.error("Missing data file {}".format(gzipped_f))
  147 + continue
  148 +
  149 + try:
  150 + cache_check[f] = {
  151 + 'status': True,
  152 + 'mtime': os.path.getmtime(gzipped_f)
  153 + }
  154 +
  155 + gf = gzip.open(gzipped_f, 'rb')
  156 + ncdata = gf.read()
  157 +
  158 + dds = nc.Dataset("in-mem-file", mode='r', memory=ncdata)
  159 + except:
  160 + logging.error("Cannot load or extract data file {}".format(gzipped_f))
  161 + cache_check[f]['status'] = False
  162 + continue
  163 +
  164 + for v in ['Time', 'StartTime', 'StopTime']:
  165 + if v not in dds.variables:
  166 + logging.error("Missing {} variable in data file {}".format(v, gzipped_f))
  167 + cache_check[f]['status'] = False
  168 + continue
  169 +
  170 + times = []
  171 + for t in np.array(dds.variables["Time"]):
  172 + if dds['Time'].dtype == np.float64:
  173 + # TimeStamp
  174 + t_str = datetime_to_ddtime(datetime.utcfromtimestamp(t))
  175 + else:
  176 + # DDTime
  177 + t_str = "".join([k.decode("UTF-8") for k in t])
  178 + times.append(int(t_str))
  179 +
  180 + if len(times) != len(set(times)):
  181 + logging.warning("Duplicate times in {}".format(gzipped_f))
  182 + cache_check[f]['status'] = False
  183 +
  184 + if not is_sorted(times):
  185 + logging.warning("Time not sorted in {}".format(gzipped_f))
  186 + cache_check[f]['status'] = False
  187 +
  188 + for t in times:
  189 + if t < start or t > stop:
  190 + logging.warning("Time outside [StartTime, StopTime] detected in {}".format(gzipped_f))
  191 + cache_check[f]['status'] = False
  192 + break
  193 +
  194 + with open(cachefile, 'wb') as handle:
  195 + pickle.dump(cache_check, handle, protocol=pickle.HIGHEST_PROTOCOL)
  196 +
  197 + return True
  198 +
  199 +
  200 +def check_ddbase(ddsys, vi=None, cachedir=None):
  201 + tree=etree.parse(ddsys)
  202 + for e in tree.iter(tag="VI"):
  203 + if vi and e.find("NAME").text != vi:
  204 + continue
  205 + if not vi and e.find("NAME").text in vi_to_exclude:
  206 + continue
  207 + check_vi(e, cachedir)
  208 +
  209 +
  210 +if __name__ == '__main__':
  211 + logging.basicConfig(format='[%(levelname)s] %(message)s', level=logging.INFO)
  212 + ddbase = '/data1/DDBASE'
  213 + ddsys = os.path.join(ddbase, 'DATA', 'DDsys.xml')
  214 + cachedir = os.path.join(os.path.dirname(__file__), '.cache')
  215 + if not os.path.exists(cachedir):
  216 + os.makedirs(cachedir)
  217 + vi = None
  218 + if len(sys.argv) > 1:
  219 + vi = sys.argv[1]
  220 + check_ddbase(ddsys, vi=vi, cachedir=cachedir)
... ...