From 8c0d5a9fca52074833cd92b5d2a0d33f3afa0021 Mon Sep 17 00:00:00 2001
From: Elena.Budnik <ebudnik@irap.omp.eu>
Date: Mon, 11 Dec 2017 14:45:47 +0100
Subject: [PATCH] correct location for NumOutput.html + scripts
---
scripts/MakeRemoteProxy.sh | 8 ++++++++
scripts/RemoveVI.sh | 20 ++++++++++++++++++++
src/REMOTEDATA/CDAWEB.php | 121 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------------------------------
src/REMOTEDATA/RemoteDataCenterClass.php | 10 ++++++----
4 files changed, 101 insertions(+), 58 deletions(-)
create mode 100755 scripts/MakeRemoteProxy.sh
create mode 100755 scripts/RemoveVI.sh
diff --git a/scripts/MakeRemoteProxy.sh b/scripts/MakeRemoteProxy.sh
new file mode 100755
index 0000000..01652d0
--- /dev/null
+++ b/scripts/MakeRemoteProxy.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+
+BASEDIR=$(dirname "$0")
+
+. ${BASEDIR}/DDServer.env
+
+php ${REMOTEDATA}/makeProxy.php
+
diff --git a/scripts/RemoveVI.sh b/scripts/RemoveVI.sh
new file mode 100755
index 0000000..a6e8452
--- /dev/null
+++ b/scripts/RemoveVI.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+if [ $# -eq 0 ]
+ then
+ echo "ViId [BaseId] should be defined"
+ exit
+ fi
+
+BASEDIR=$(dirname "$0")
+
+. ${BASEDIR}/DDServer.env
+
+if [ $# -eq 1 ]
+ then
+ php ${DATAMANAGER}/RemoveVI.php $1
+ else
+ php ${DATAMANAGER}/RemoveVI.php $1 $2
+fi
+
+
diff --git a/src/REMOTEDATA/CDAWEB.php b/src/REMOTEDATA/CDAWEB.php
index 64da79f..eb6ef48 100644
--- a/src/REMOTEDATA/CDAWEB.php
+++ b/src/REMOTEDATA/CDAWEB.php
@@ -26,7 +26,7 @@ class CDAWEB extends RemoteDataCenterClass
// not in "https://heliophysicsdata.sci.gsfc.nasa.gov/queries/CDAWeb_SPASE.xql" List;
// FOR INFO : Excluded automatically
- private $excludeDatasets = ["DMSP_R0_SSJ4", "DMSP_R0_SSIES", "I7_R0_LEPEDEA", "I8_R0_LEPEDEA" ];
+ private $excludeDatasets = ["DMSP_R0_SSJ4", "DMSP_R0_SSIES", "I7_R0_LEPEDEA", "I8_R0_LEPEDEA", "WI_STRAHL0_SWE" ];
protected $DDserverXml, $DDserverXmlName;
@@ -39,9 +39,9 @@ class CDAWEB extends RemoteDataCenterClass
}
public function init()
- {
-
+ {
error_log("CDAWEB Proxy creation on ".date("Y-m-d").PHP_EOL,3,log);
+ error_log("CDAWEB Proxy creation on ".date("Y-m-d").PHP_EOL,3,err);
$this->getAllSpaseDatasets();
}
@@ -95,8 +95,10 @@ class CDAWEB extends RemoteDataCenterClass
if ( $observatories->length == 0 )
{
echo ' No observatories '.PHP_EOL;
- if ($obsGroup->getElementsByTagName("InstrumentDescription")->length > 0)
- echo ' CHECK INSTRUMENTS '.PHP_EOL;
+ if ($obsGroup->getElementsByTagName("InstrumentDescription")->length > 0) {
+ echo ' CHECK INSTRUMENTS '.PHP_EOL;
+ error_log(" CHECK INSTRUMENTS ".$obsGroup.PHP_EOL, 3, err);
+ }
continue;
}
$obsIds = array();
@@ -135,9 +137,7 @@ class CDAWEB extends RemoteDataCenterClass
}
protected function getDatasetByObservatoryAndInstrument($obsGroupId, $obsId, $insId)
- {
- // echo "In getDataset ".$obsId." ".$insId.PHP_EOL;
-
+ {
$grpEncodedId = $this->html_encode($obsGroupId);
$insEncodedId = $this->html_encode($insId);
$obsEncodedId = $this->html_encode($obsId);
@@ -148,7 +148,6 @@ class CDAWEB extends RemoteDataCenterClass
curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets?observatory=$obsEncodedId&instrument=$insEncodedId");
$res->loadXML(curl_exec($this->ch));
-exit ($res->saveXML());
$datasets = $res->getElementsByTagName('DatasetDescription');
$dsIds = array();
@@ -156,7 +155,7 @@ exit ($res->saveXML());
foreach ( $datasets as $ds)
{
// $obs = $ds->getElementsByTagName('Observatory')->item(0)->nodeValue;
- if ($obs != $obsId ) continue;
+ // if ($obs != $obsId ) continue;
$dsIds[] = $ds->getElementsByTagName('Id')->item(0)->nodeValue;
}
@@ -210,11 +209,15 @@ exit ($res->saveXML());
$this->updateDDServerXml("GlobalStop",$endTime);
// no general description - strange dataset
- if ($sampling == -1)
+ if ($sampling == -1) {
+ error_log("! No general description : ".$dsId.PHP_EOL,3,err);
return null;
+ }
- if ($sampling < -1)
+ if ($sampling < -1) {
echo " !!! $dsId $sampling".PHP_EOL;
+ error_log("! No sampling : ".$dsId." : ".$sampling.PHP_EOL,3,err);
+ }
else
$this->updateDDServerXml("MinSampling",$sampling);
@@ -308,14 +311,14 @@ exit ($res->saveXML());
{
require_once "simple_html_dom.php";
- if (file_exists("NumericalData.html"))
- rename("NumericalData.html","NumericalData.html.bak");
+ if (file_exists($this->location."/NumericalData.html"))
+ rename($this->location."/NumericalData.html",$this->location."/NumericalData.html.bak");
// if cannot reach CDAWEB Spase Registry use an old file
- if (!copy(CDAWebConfigClass::$spaseRegistry, "NumericalData.html"))
- copy("NumericalData.html.bak", "NumericalData.html");
+ if (!copy(CDAWebConfigClass::$spaseRegistry, $this->location."/NumericalData.html"))
+ copy($this->location."/NumericalData.html.bak", $this->location."/NumericalData.html");
- $html = file_get_html('NumericalData.html');
+ $html = file_get_html($this->location.'/NumericalData.html');
$ids = $html->find('td[class="Spase.URL.ProductID"]');
foreach ($ids as $id)
@@ -324,7 +327,7 @@ exit ($res->saveXML());
$key = $id->find('a');
$this->CDAWEB[$key[0]->innertext] = $ref[0]->innertext;
}
-
+ error_log("SPASE-defined CDAWEB datasets from ".CDAWebConfigClass::$spaseRegistry." : ".count($this->CDAWEB).PHP_EOL,3,log);
echo "SPASE-defined CDAWEB datasets : ".count($this->CDAWEB).PHP_EOL;
}
@@ -363,7 +366,7 @@ exit ($res->saveXML());
if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
$exists = false;
- // echo " NO master cdf ".$dsId.PHP_EOL;
+ error_log("INFO: no master CDF ".$dsId.PHP_EOL,3,log);
}
else {
$exists = true;
@@ -392,10 +395,11 @@ exit ($res->saveXML());
$res = new DomDocument("1.0");
$res->loadXML(curl_exec($this->ch));
- //TODO errors
+
if ($res->getElementsByTagName("html")->length > 0)
{
- echo $res->saveXML();
+ error_log("ERROR no response for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
+ error_log($res->saveXML(),3,err);
exit();
}
@@ -412,7 +416,7 @@ exit ($res->saveXML());
$file_headers = @get_headers($url);
if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
- //TODO errors
+ error_log("ERROR 404 Not Found for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
continue;
}
$temp = explode('/',$url);
@@ -434,7 +438,7 @@ exit ($res->saveXML());
}
else
{
- //TODO errors
+ error_log("ERROR cannot copy files : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
}
}
@@ -445,11 +449,13 @@ exit ($res->saveXML());
protected function convert2nc($file)
{
- //TODO errors
system("cdfnew2nc $file");
$ncFile = str_replace(".cdf", ".nc", $file);
- if (!file_exists($ncFile)) return false;
+ if (!file_exists($ncFile)) {
+ error_log("ERROR while converting ".$file.PHP_EOL,3,err);
+ return false;
+ }
unlink($file);
return $ncFile;
@@ -479,28 +485,27 @@ exit ($res->saveXML());
$this->domAmda->appendChild($dataRoot);
$this->dataCenter=$this->domAmda->createElement('dataCenter');
-
- $this->dataCenter->setAttribute('xml:id', $this->baseID);
- // $this->dataCenter->setIdAttribute('xml:id', true);
- $this->dataCenter->setAttribute('name', $this->baseID);
-
+ $this->setDataCenterAttributes();
+
$dataRoot->appendChild($this->dataCenter);
-
- $this->setDataCenterAttributes();
+
$first = true;
- foreach ($this->obsGroupsIds as $insType => $groupIds){
+ foreach ($this->obsGroupsIds as $insType => $groupIds)
+ {
// getElementByID doesn't work on newly created Document
if (!$first) {
$this->domAmda->loadXML($this->domAmda->saveXML());
$this->dataCenter = $this->domAmda->getElementById($this->baseID);
}
- foreach ($groupIds as $groupId => $obss){
+ foreach ($groupIds as $groupId => $obss)
+ {
$obsNodes = array();
-
+
foreach ($obss as $obs => $inss)
- {
+ {
+ // ! observatory OMNI is taken for observatoriesGroup OMNI only !
if (substr($obs,0,4) == "OMNI" && substr($groupId,0,4) != "OMNI"){
echo 'OMNI '.$obs.' '.$missionId.PHP_EOL;
continue;
@@ -508,10 +513,12 @@ exit ($res->saveXML());
$insNodes = array();
// Create instrument nodes
- foreach ($inss as $ins => $dss){
+ foreach ($inss as $ins => $dss)
+ {
$dsNodes = array();
// Create datasets nodes
- foreach ($dss as $ds){
+ foreach ($dss as $ds)
+ {
$dsId = $this->baseID.":".$ds;
if (!$this->domAmda->getElementById($dsId)){
$this->initDDServerXml($ds,$ins,$obs);
@@ -522,7 +529,8 @@ exit ($res->saveXML());
}
}
} // foreach ($dss as $ds)
- if ($dsNodes){
+ if ($dsNodes)
+ {
// last $spase_res : instrument should be the same
$insSpaseId = $this->getInstrumentSpase();
if (!$insSpaseId )
@@ -543,16 +551,13 @@ exit ($res->saveXML());
}
}
} // foreach ($inss as $ins => $dss)
- if ($insNodes){
-
+ if ($insNodes)
+ {
$obsId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId);
if (!($obsNode = $this->domAmda->getElementById($obsId))){
$obsNode = $this->createObservatoryNode($obsSpaseId, $groupId);
-
-// if (!$first){
-// echo "NEW OBS ".$obsId.PHP_EOL;
-// }
+
$obsNodes[] = $obsNode;
}
@@ -562,7 +567,8 @@ exit ($res->saveXML());
}
} // foreach ($obss as $obs => $inss)
- if ($obsNodes){
+ if ($obsNodes)
+ {
$missionId = $this->baseID.":".$this->param2dd($groupId);
if (!($missionNode = $this->domAmda->getElementById($missionId))){
@@ -570,9 +576,10 @@ exit ($res->saveXML());
$this->dataCenter->appendChild($missionNode);
}
- foreach ($obsNodes as $obsNode){
+ foreach ($obsNodes as $obsNode)
+ {
// observatory == mission
- if ($obsNode->getAttribute('name') == $missionNode->getAttribute('name')) {
+ if (strtolower($obsNode->getAttribute('name')) == strtolower($missionNode->getAttribute('name'))) {
$insNodes = $obsNode->getElementsByTagName("instrument");
foreach ($insNodes as $insNode)
$missionNode->appendChild($insNode);
@@ -588,7 +595,8 @@ exit ($res->saveXML());
$this->closeConnection();
}
- protected function createMissionNode($groupId){
+ protected function createMissionNode($groupId)
+ {
$missionId = $this->baseID.":".$this->param2dd($groupId);
$missionNode = $this->domAmda->createElement('mission');
$missionNode->setAttribute("xml:id",$missionId);
@@ -597,8 +605,8 @@ exit ($res->saveXML());
return $missionNode;
}
- protected function getInstrumentSpase(){
-
+ protected function getInstrumentSpase()
+ {
$this->insXML = new DomDocument("1.0");
if ($this->spase_res){
@@ -619,8 +627,8 @@ exit ($res->saveXML());
return null;
}
- protected function getObservatorySpase(){
-
+ protected function getObservatorySpase()
+ {
if ($this->insXML){
$observatory = $this->insXML->getElementsByTagName('ObservatoryID');
@@ -632,7 +640,12 @@ exit ($res->saveXML());
return null;
}
- protected function setDataCenterAttributes(){}
+ protected function setDataCenterAttributes(){
+ $this->dataCenter->setAttribute('xml:id', $this->baseID);
+ // $this->dataCenter->setIdAttribute('xml:id', true);
+ $this->dataCenter->setAttribute('name', $this->baseID);
+ }
+
protected function makeArgumentsList(){}
}
diff --git a/src/REMOTEDATA/RemoteDataCenterClass.php b/src/REMOTEDATA/RemoteDataCenterClass.php
index 7c19d5c..4ad70a4 100644
--- a/src/REMOTEDATA/RemoteDataCenterClass.php
+++ b/src/REMOTEDATA/RemoteDataCenterClass.php
@@ -15,6 +15,7 @@ abstract class RemoteDataCenterClass
public $domAmdaName = "base.xml";
public $additionalArgs = null;
public $baseID = null;
+ public $location;
public $tree = null;
protected $DDserverDir;
@@ -27,7 +28,8 @@ abstract class RemoteDataCenterClass
*/
function __construct()
{
- $this->baseID = get_class($this);
+ $this->baseID = get_class($this);
+ $this->location = RemoteData."/bases/".$this->baseID;
}
/* Function to change External Bases stuff in case of existing AMDA aliases.
@@ -56,7 +58,7 @@ abstract class RemoteDataCenterClass
public function saveProxy()
{
- $this->domAmda->save(RemoteData."/bases/".$this->baseID."/".$this->domAmdaName);
+ $this->domAmda->save($this->location."/".$this->domAmdaName);
}
abstract public function init();
@@ -72,8 +74,8 @@ abstract class RemoteDataCenterClass
public function initProxy()
{
- if (!is_dir(RemoteData."/bases/".$this->baseID))
- mkdir(RemoteData."/bases/".$this->baseID, 0755, true);
+ if (!is_dir($this->location))
+ mkdir($this->location, 0755, true);
$this->DDserverDir = RemoteData."/DDServer/".$this->baseID;
--
libgit2 0.21.2