CDPP / DDServer

Commit 8c0d5a9fca52074833cd92b5d2a0d33f3afa0021

Authored by Elena.Budnik
1 parent 05fbcab0
Exists in master and in 9 other branches WIP, clean_real, create-user, epntap, evol_9042, fix-cdaweb, sqlite_test, time-in-double, ubuntu

correct location for NumOutput.html + scripts

Showing 4 changed files with 101 additions and 58 deletions   Show diff stats
Inline Side-by-side
scripts/MakeRemoteProxy.sh 0 → 100755
  Diff comments   View file @8c0d5a9
... ... @@ -0,0 +1,8 @@
  1 +#!/bin/sh
  2 +
  3 +BASEDIR=$(dirname "$0")
  4 +
  5 +. ${BASEDIR}/DDServer.env
  6 +
  7 +php ${REMOTEDATA}/makeProxy.php
  8 +
... ...
scripts/RemoveVI.sh 0 → 100755
  Diff comments   View file @8c0d5a9
... ... @@ -0,0 +1,20 @@
  1 +#!/bin/sh
  2 +
  3 +if [ $# -eq 0 ]
  4 + then
  5 + echo "ViId [BaseId] should be defined"
  6 + exit
  7 + fi
  8 +
  9 +BASEDIR=$(dirname "$0")
  10 +
  11 +. ${BASEDIR}/DDServer.env
  12 +
  13 +if [ $# -eq 1 ]
  14 + then
  15 + php ${DATAMANAGER}/RemoveVI.php $1
  16 + else
  17 + php ${DATAMANAGER}/RemoveVI.php $1 $2
  18 +fi
  19 +
  20 +
... ...
src/REMOTEDATA/CDAWEB.php
  Diff comments   View file @8c0d5a9
... ... @@ -26,7 +26,7 @@ class CDAWEB extends RemoteDataCenterClass
26 26  
27 27 // not in "https://heliophysicsdata.sci.gsfc.nasa.gov/queries/CDAWeb_SPASE.xql" List;
28 28 // FOR INFO : Excluded automatically
29   - private $excludeDatasets = ["DMSP_R0_SSJ4", "DMSP_R0_SSIES", "I7_R0_LEPEDEA", "I8_R0_LEPEDEA" ];
  29 + private $excludeDatasets = ["DMSP_R0_SSJ4", "DMSP_R0_SSIES", "I7_R0_LEPEDEA", "I8_R0_LEPEDEA", "WI_STRAHL0_SWE" ];
30 30  
31 31 protected $DDserverXml, $DDserverXmlName;
32 32  
... ... @@ -39,9 +39,9 @@ class CDAWEB extends RemoteDataCenterClass
39 39 }
40 40  
41 41 public function init()
42   - {
43   -
  42 + {
44 43 error_log("CDAWEB Proxy creation on ".date("Y-m-d").PHP_EOL,3,log);
  44 + error_log("CDAWEB Proxy creation on ".date("Y-m-d").PHP_EOL,3,err);
45 45  
46 46 $this->getAllSpaseDatasets();
47 47 }
... ... @@ -95,8 +95,10 @@ class CDAWEB extends RemoteDataCenterClass
95 95 if ( $observatories->length == 0 )
96 96 {
97 97 echo ' No observatories '.PHP_EOL;
98   - if ($obsGroup->getElementsByTagName("InstrumentDescription")->length > 0)
99   - echo ' CHECK INSTRUMENTS '.PHP_EOL;
  98 + if ($obsGroup->getElementsByTagName("InstrumentDescription")->length > 0) {
  99 + echo ' CHECK INSTRUMENTS '.PHP_EOL;
  100 + error_log(" CHECK INSTRUMENTS ".$obsGroup.PHP_EOL, 3, err);
  101 + }
100 102 continue;
101 103 }
102 104 $obsIds = array();
... ... @@ -135,9 +137,7 @@ class CDAWEB extends RemoteDataCenterClass
135 137 }
136 138  
137 139 protected function getDatasetByObservatoryAndInstrument($obsGroupId, $obsId, $insId)
138   - {
139   - // echo "In getDataset ".$obsId." ".$insId.PHP_EOL;
140   -
  140 + {
141 141 $grpEncodedId = $this->html_encode($obsGroupId);
142 142 $insEncodedId = $this->html_encode($insId);
143 143 $obsEncodedId = $this->html_encode($obsId);
... ... @@ -148,7 +148,6 @@ class CDAWEB extends RemoteDataCenterClass
148 148 curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets?observatory=$obsEncodedId&instrument=$insEncodedId");
149 149  
150 150 $res->loadXML(curl_exec($this->ch));
151   -exit ($res->saveXML());
152 151  
153 152 $datasets = $res->getElementsByTagName('DatasetDescription');
154 153 $dsIds = array();
... ... @@ -156,7 +155,7 @@ exit ($res->saveXML());
156 155 foreach ( $datasets as $ds)
157 156 {
158 157 // $obs = $ds->getElementsByTagName('Observatory')->item(0)->nodeValue;
159   - if ($obs != $obsId ) continue;
  158 + // if ($obs != $obsId ) continue;
160 159 $dsIds[] = $ds->getElementsByTagName('Id')->item(0)->nodeValue;
161 160 }
162 161  
... ... @@ -210,11 +209,15 @@ exit ($res->saveXML());
210 209 $this->updateDDServerXml("GlobalStop",$endTime);
211 210  
212 211 // no general description - strange dataset
213   - if ($sampling == -1)
  212 + if ($sampling == -1) {
  213 + error_log("! No general description : ".$dsId.PHP_EOL,3,err);
214 214 return null;
  215 + }
215 216  
216   - if ($sampling < -1)
  217 + if ($sampling < -1) {
217 218 echo " !!! $dsId $sampling".PHP_EOL;
  219 + error_log("! No sampling : ".$dsId." : ".$sampling.PHP_EOL,3,err);
  220 + }
218 221 else
219 222 $this->updateDDServerXml("MinSampling",$sampling);
220 223  
... ... @@ -308,14 +311,14 @@ exit ($res-&gt;saveXML());
308 311 {
309 312 require_once "simple_html_dom.php";
310 313  
311   - if (file_exists("NumericalData.html"))
312   - rename("NumericalData.html","NumericalData.html.bak");
  314 + if (file_exists($this->location."/NumericalData.html"))
  315 + rename($this->location."/NumericalData.html",$this->location."/NumericalData.html.bak");
313 316  
314 317 // if cannot reach CDAWEB Spase Registry use an old file
315   - if (!copy(CDAWebConfigClass::$spaseRegistry, "NumericalData.html"))
316   - copy("NumericalData.html.bak", "NumericalData.html");
  318 + if (!copy(CDAWebConfigClass::$spaseRegistry, $this->location."/NumericalData.html"))
  319 + copy($this->location."/NumericalData.html.bak", $this->location."/NumericalData.html");
317 320  
318   - $html = file_get_html('NumericalData.html');
  321 + $html = file_get_html($this->location.'/NumericalData.html');
319 322 $ids = $html->find('td[class="Spase.URL.ProductID"]');
320 323  
321 324 foreach ($ids as $id)
... ... @@ -324,7 +327,7 @@ exit ($res-&gt;saveXML());
324 327 $key = $id->find('a');
325 328 $this->CDAWEB[$key[0]->innertext] = $ref[0]->innertext;
326 329 }
327   -
  330 + error_log("SPASE-defined CDAWEB datasets from ".CDAWebConfigClass::$spaseRegistry." : ".count($this->CDAWEB).PHP_EOL,3,log);
328 331 echo "SPASE-defined CDAWEB datasets : ".count($this->CDAWEB).PHP_EOL;
329 332 }
330 333  
... ... @@ -363,7 +366,7 @@ exit ($res-&gt;saveXML());
363 366  
364 367 if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
365 368 $exists = false;
366   - // echo " NO master cdf ".$dsId.PHP_EOL;
  369 + error_log("INFO: no master CDF ".$dsId.PHP_EOL,3,log);
367 370 }
368 371 else {
369 372 $exists = true;
... ... @@ -392,10 +395,11 @@ exit ($res-&gt;saveXML());
392 395 $res = new DomDocument("1.0");
393 396  
394 397 $res->loadXML(curl_exec($this->ch));
395   - //TODO errors
  398 +
396 399 if ($res->getElementsByTagName("html")->length > 0)
397 400 {
398   - echo $res->saveXML();
  401 + error_log("ERROR no response for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
  402 + error_log($res->saveXML(),3,err);
399 403 exit();
400 404 }
401 405  
... ... @@ -412,7 +416,7 @@ exit ($res-&gt;saveXML());
412 416 $file_headers = @get_headers($url);
413 417  
414 418 if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
415   - //TODO errors
  419 + error_log("ERROR 404 Not Found for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
416 420 continue;
417 421 }
418 422 $temp = explode('/',$url);
... ... @@ -434,7 +438,7 @@ exit ($res-&gt;saveXML());
434 438 }
435 439 else
436 440 {
437   - //TODO errors
  441 + error_log("ERROR cannot copy files : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
438 442 }
439 443 }
440 444  
... ... @@ -445,11 +449,13 @@ exit ($res-&gt;saveXML());
445 449  
446 450 protected function convert2nc($file)
447 451 {
448   - //TODO errors
449 452 system("cdfnew2nc $file");
450 453 $ncFile = str_replace(".cdf", ".nc", $file);
451 454  
452   - if (!file_exists($ncFile)) return false;
  455 + if (!file_exists($ncFile)) {
  456 + error_log("ERROR while converting ".$file.PHP_EOL,3,err);
  457 + return false;
  458 + }
453 459 unlink($file);
454 460  
455 461 return $ncFile;
... ... @@ -479,28 +485,27 @@ exit ($res-&gt;saveXML());
479 485 $this->domAmda->appendChild($dataRoot);
480 486  
481 487 $this->dataCenter=$this->domAmda->createElement('dataCenter');
482   -
483   - $this->dataCenter->setAttribute('xml:id', $this->baseID);
484   - // $this->dataCenter->setIdAttribute('xml:id', true);
485   - $this->dataCenter->setAttribute('name', $this->baseID);
486   -
  488 + $this->setDataCenterAttributes();
  489 +
487 490 $dataRoot->appendChild($this->dataCenter);
488   -
489   - $this->setDataCenterAttributes();
  491 +
490 492 $first = true;
491 493  
492   - foreach ($this->obsGroupsIds as $insType => $groupIds){
  494 + foreach ($this->obsGroupsIds as $insType => $groupIds)
  495 + {
493 496 // getElementByID doesn't work on newly created Document
494 497 if (!$first) {
495 498 $this->domAmda->loadXML($this->domAmda->saveXML());
496 499 $this->dataCenter = $this->domAmda->getElementById($this->baseID);
497 500 }
498   - foreach ($groupIds as $groupId => $obss){
499 501  
  502 + foreach ($groupIds as $groupId => $obss)
  503 + {
500 504 $obsNodes = array();
501   -
  505 +
502 506 foreach ($obss as $obs => $inss)
503   - {
  507 + {
  508 + // ! observatory OMNI is taken for observatoriesGroup OMNI only !
504 509 if (substr($obs,0,4) == "OMNI" && substr($groupId,0,4) != "OMNI"){
505 510 echo 'OMNI '.$obs.' '.$missionId.PHP_EOL;
506 511 continue;
... ... @@ -508,10 +513,12 @@ exit ($res-&gt;saveXML());
508 513  
509 514 $insNodes = array();
510 515 // Create instrument nodes
511   - foreach ($inss as $ins => $dss){
  516 + foreach ($inss as $ins => $dss)
  517 + {
512 518 $dsNodes = array();
513 519 // Create datasets nodes
514   - foreach ($dss as $ds){
  520 + foreach ($dss as $ds)
  521 + {
515 522 $dsId = $this->baseID.":".$ds;
516 523 if (!$this->domAmda->getElementById($dsId)){
517 524 $this->initDDServerXml($ds,$ins,$obs);
... ... @@ -522,7 +529,8 @@ exit ($res-&gt;saveXML());
522 529 }
523 530 }
524 531 } // foreach ($dss as $ds)
525   - if ($dsNodes){
  532 + if ($dsNodes)
  533 + {
526 534 // last $spase_res : instrument should be the same
527 535 $insSpaseId = $this->getInstrumentSpase();
528 536 if (!$insSpaseId )
... ... @@ -543,16 +551,13 @@ exit ($res-&gt;saveXML());
543 551 }
544 552 }
545 553 } // foreach ($inss as $ins => $dss)
546   - if ($insNodes){
547   -
  554 + if ($insNodes)
  555 + {
548 556 $obsId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId);
549 557  
550 558 if (!($obsNode = $this->domAmda->getElementById($obsId))){
551 559 $obsNode = $this->createObservatoryNode($obsSpaseId, $groupId);
552   -
553   -// if (!$first){
554   -// echo "NEW OBS ".$obsId.PHP_EOL;
555   -// }
  560 +
556 561 $obsNodes[] = $obsNode;
557 562 }
558 563  
... ... @@ -562,7 +567,8 @@ exit ($res-&gt;saveXML());
562 567 }
563 568 } // foreach ($obss as $obs => $inss)
564 569  
565   - if ($obsNodes){
  570 + if ($obsNodes)
  571 + {
566 572 $missionId = $this->baseID.":".$this->param2dd($groupId);
567 573  
568 574 if (!($missionNode = $this->domAmda->getElementById($missionId))){
... ... @@ -570,9 +576,10 @@ exit ($res-&gt;saveXML());
570 576 $this->dataCenter->appendChild($missionNode);
571 577 }
572 578  
573   - foreach ($obsNodes as $obsNode){
  579 + foreach ($obsNodes as $obsNode)
  580 + {
574 581 // observatory == mission
575   - if ($obsNode->getAttribute('name') == $missionNode->getAttribute('name')) {
  582 + if (strtolower($obsNode->getAttribute('name')) == strtolower($missionNode->getAttribute('name'))) {
576 583 $insNodes = $obsNode->getElementsByTagName("instrument");
577 584 foreach ($insNodes as $insNode)
578 585 $missionNode->appendChild($insNode);
... ... @@ -588,7 +595,8 @@ exit ($res-&gt;saveXML());
588 595 $this->closeConnection();
589 596 }
590 597  
591   - protected function createMissionNode($groupId){
  598 + protected function createMissionNode($groupId)
  599 + {
592 600 $missionId = $this->baseID.":".$this->param2dd($groupId);
593 601 $missionNode = $this->domAmda->createElement('mission');
594 602 $missionNode->setAttribute("xml:id",$missionId);
... ... @@ -597,8 +605,8 @@ exit ($res-&gt;saveXML());
597 605 return $missionNode;
598 606 }
599 607  
600   - protected function getInstrumentSpase(){
601   -
  608 + protected function getInstrumentSpase()
  609 + {
602 610 $this->insXML = new DomDocument("1.0");
603 611  
604 612 if ($this->spase_res){
... ... @@ -619,8 +627,8 @@ exit ($res-&gt;saveXML());
619 627 return null;
620 628 }
621 629  
622   - protected function getObservatorySpase(){
623   -
  630 + protected function getObservatorySpase()
  631 + {
624 632 if ($this->insXML){
625 633 $observatory = $this->insXML->getElementsByTagName('ObservatoryID');
626 634  
... ... @@ -632,7 +640,12 @@ exit ($res-&gt;saveXML());
632 640 return null;
633 641 }
634 642  
635   - protected function setDataCenterAttributes(){}
  643 + protected function setDataCenterAttributes(){
  644 + $this->dataCenter->setAttribute('xml:id', $this->baseID);
  645 + // $this->dataCenter->setIdAttribute('xml:id', true);
  646 + $this->dataCenter->setAttribute('name', $this->baseID);
  647 + }
  648 +
636 649 protected function makeArgumentsList(){}
637 650  
638 651 }
... ...
src/REMOTEDATA/RemoteDataCenterClass.php
  Diff comments   View file @8c0d5a9
... ... @@ -15,6 +15,7 @@ abstract class RemoteDataCenterClass
15 15 public $domAmdaName = "base.xml";
16 16 public $additionalArgs = null;
17 17 public $baseID = null;
  18 + public $location;
18 19  
19 20 public $tree = null;
20 21 protected $DDserverDir;
... ... @@ -27,7 +28,8 @@ abstract class RemoteDataCenterClass
27 28 */
28 29 function __construct()
29 30 {
30   - $this->baseID = get_class($this);
  31 + $this->baseID = get_class($this);
  32 + $this->location = RemoteData."/bases/".$this->baseID;
31 33 }
32 34  
33 35 /* Function to change External Bases stuff in case of existing AMDA aliases.
... ... @@ -56,7 +58,7 @@ abstract class RemoteDataCenterClass
56 58  
57 59 public function saveProxy()
58 60 {
59   - $this->domAmda->save(RemoteData."/bases/".$this->baseID."/".$this->domAmdaName);
  61 + $this->domAmda->save($this->location."/".$this->domAmdaName);
60 62 }
61 63  
62 64 abstract public function init();
... ... @@ -72,8 +74,8 @@ abstract class RemoteDataCenterClass
72 74  
73 75 public function initProxy()
74 76 {
75   - if (!is_dir(RemoteData."/bases/".$this->baseID))
76   - mkdir(RemoteData."/bases/".$this->baseID, 0755, true);
  77 + if (!is_dir($this->location))
  78 + mkdir($this->location, 0755, true);
77 79  
78 80 $this->DDserverDir = RemoteData."/DDServer/".$this->baseID;
79 81  
... ...