"%20"); return strtr($param,$pairs); } public function html_decode($param) { $pairs = array("%20" => " "); return strtr($param,$pairs); } public function init() { error_log("CDAWEB Proxy creation on ".date("Y-m-d\TH:i:s").PHP_EOL,3,log); error_log("CDAWEB Proxy creation on ".date("Y-m-d\TH:i:s").PHP_EOL,3,err); error_reporting(ALL); $this->initStreamContext(); $this->getAllSpaseDatasets(); // predefined samplings if (file_exists(RemoteData."/samplingCDAWEB.xml")) { $samplingsDom = new DomDocument("1.0"); if ($samplingsDom->load(RemoteData."/samplingCDAWEB.xml")) { $this->xp = new domxpath($samplingsDom); } } } private function openConnection() { $this->ch = curl_init(); curl_setopt($this->ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($this->ch, CURLOPT_TIMEOUT, 600); // Add proxy definition $PROXY_HOST=getenv('PROXY_HOST'); $PROXY_USERPWD=getenv('PROXY_USERPWD'); if (!empty($PROXY_HOST)) { curl_setopt($this->ch, CURLOPT_PROXY, getenv('PROXY_HOST')); if (!empty($PROXY_USERPWD)) { curl_setopt($this->ch, CURLOPT_PROXYUSERPWD, getenv('PROXY_USERPWD')); } } } private function closeConnection() { curl_close($this->ch); } private function initStreamContext() { $context = array( 'http' => array( 'timeout' => 600, ), ); $PROXY_HOST=getenv('PROXY_HOST'); $PROXY_USERPWD=getenv('PROXY_USERPWD'); if (!empty($PROXY_HOST)) { $context['http']['proxy'] = "tcp://$PROXY_HOST"; $context['http']['request_fulluri'] = TRUE; if (!empty($PROXY_USERPWD)) { $context['http']['header'] = "Proxy-Authorization: Basic ".base64_encode($PROXY_USERPWD); } } stream_context_set_default($context); } /** * Using CDAS REST APIs get ALL CDAWEB dataset descriptions * Create an array obsGroupsIds[$obsGroupId][$obsId][$insId][$dsId] */ protected function getRemoteTree() { $this->openConnection(); $this->obsGroupsIds = array(); $dom = NULL; $reuse_cache_file = FALSE; if (file_exists($this->location."/datasets.xml")) { if (time() - filemtime($this->location."/datasets.xml") < 86400) { $dom = new DOMDocument(); if ($dom->load($this->location."/datasets.xml")) { echo "Re-use cache file ".$this->location."/datasets.xml".PHP_EOL; $reuse_cache_file = TRUE; } } } if (!$reuse_cache_file) { $dom = $this->loadFromCDAWebWS("datasets", array()); if (!$dom) { error_log('Cannot retrieve CDAWeb datasets list'); return; } $dom->save($this->location."/datasets.xml"); } $datasets = $dom->getElementsByTagName("DatasetDescription"); echo "All CDAWeb datasets : ".$datasets->length.PHP_EOL; foreach ($datasets as $ds) { $insType = $ds->getElementsByTagName("InstrumentType")->item(0)->nodeValue; if (in_array ($this->html_encode($insType), $this->validInstrumentTypes )) { $id = $ds->getElementsByTagName("Id")->item(0)->nodeValue; $insId = $ds->getElementsByTagName("Instrument")->item(0)->nodeValue; if (! in_array ($insId, $this->excludeIns )) { $obsId = "UNKNOWN"; if ($ds->getElementsByTagName("Observatory")->length > 0) $obsId = $ds->getElementsByTagName("Observatory")->item(0)->nodeValue; $obsGroupId = "UNKNOWN"; if ($ds->getElementsByTagName("ObservatoryGroup")->length > 0) $obsGroupId = $ds->getElementsByTagName("ObservatoryGroup")->item(0)->nodeValue; if ($obsId == "UNKNOWN") { $obsId = $obsGroupId; } if ( $obsGroupId == "UNKNOWN" ) { $obsGroupId = $obsId; } if ( $obsGroupId != "UNKNOWN" ) { $this->obsGroupsIds[$obsGroupId][$obsId][$insId][] = $id; } } } } } protected function createObservatoryNode($id, $grpId) { $obsNode = $this->domAmda->createElement('observatory'); $obsNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($grpId).":".$this->param2dd($id)); $obsNode->setAttribute("name",$id); return $obsNode; } protected function createInstrumentNode($id, $obsId, $groupId) { $insNode = $this->domAmda->createElement('instrument'); $insNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsId).":".$id); $insNode->setAttribute("name",$id); // $insNode->setAttribute("description",$ins[1]); return $insNode; } protected function createDatasetNode($dsId) { if ( !array_key_exists($dsId, $this->CDAWEB )) return null; // no description in SpaseRegistry => we do not add this dataset $dsNode = $this->domAmda->createElement('dataset'); $obj = $this->loadFromCDAWebWS("datasets", array("idPattern" => $dsId), TRUE); if (!$obj) { return NULL; } $dataSet = $obj->DatasetDescription; $dsNode->setAttribute("xml:id",$this->baseID.":".$dsId); $dsNode->setAttribute("name", $dsId); $startTime = $dataSet[0]->TimeInterval->Start; $endTime = $dataSet[0]->TimeInterval->End; $label = $dataSet[0]->Label; $url = $dataSet[0]->DatasetLink[0]->Url; $notesUrl = $dataSet[0]->Notes; $piAffilation = $dataSet[0]->PiAffiliation; $piName = $dataSet[0]->PiName; $sampling = $this->getDatasetSpaseDescription($dsId); $this->updateDDServerXml("GlobalStart",$startTime); $this->updateDDServerXml("GlobalStop",$endTime); // no SPASE description if ($sampling == -1) return null; if ($sampling <= 0) { // check if there is file with predefined samplings if ($this->xp) { $set = $this->xp->query("//dataset[@xml:id='".$dsId."']"); if ($set->length > 0) { $sampling = $set->item(0)->nodeValue; echo "from samplingCDAWEB : ".$dsId." : ".$sampling.PHP_EOL; $this->updateDDServerXml("MinSampling",$sampling); } else { echo " No sampling : ".$dsId.PHP_EOL; error_log("! No sampling and No dataset in samplingCDAWEB defined : ".$dsId." : ".$sampling.PHP_EOL,3,err); } } else { echo " No xpath ".PHP_EOL; error_log("! No sampling and No xpath defined : ".$dsId." : ".$sampling.PHP_EOL,3,err); } } else $this->updateDDServerXml("MinSampling",$sampling); $dsNode->setAttribute('spaseUrl',$this->CDAWEB[$dsId]); // $dsNode->setAttribute('masterCdf',$this->existsMasterCdf($dsId)); $dsNode->setAttribute("desc", "$label; $startTime - $endTime"); $dsNode->setAttribute("dataStart", $startTime); $dsNode->setAttribute("dataStop", $endTime); $parameterNodes = $this->createParameterNodes($dsId); foreach ($parameterNodes as $parameterNode) { $dsNode->appendChild($parameterNode); } return $dsNode; } protected function createParameterNodes($dsId) { $obj = $this->loadFromCDAWebWS("datasets/".$dsId."/variables", array(), TRUE); if (!$obj) return array(); $parameters = $obj->VariableDescription; $paramNodes = array(); foreach ($parameters as $param) { $paramNode = $this->domAmda->createElement('parameter'); $paramNode->setAttribute("xml:id", $this->baseID.":".$dsId.":".$param->Name); $paramNode->setAttribute("name", $param->Name); $paramNodes[] = $paramNode; } return $paramNodes; } protected function getDatasetSpaseDescription($dsID) { if ( !array_key_exists($dsID, $this->CDAWEB )) return -1; // no description in SpaseRegistry if (!$this->loadSpaseResource($this->CDAWEB[$dsID], $this->spase_res)) return -100; $messages = $this->spase_res->getElementsByTagName('Message'); if ($messages->length > 0) { // foreach ($messages as $message) // echo $message->nodeValue.PHP_EOL; return -2; // no description in www-spase } // $instrument = $this->spase_res->getElementsByTagName('InstrumentID'); // echo " SPASE : instrument ".$instrument->item(0)->nodeValue.PHP_EOL; $cadence = $this->spase_res->getElementsByTagName('Cadence'); if ($cadence->length == 0) return -3; // no cadence in spase xml $sampling = $this->cadence2sampling($cadence->item(0)->nodeValue); return $sampling; } public function cadence2sampling($cadence) { $scale = array('S' => 1, 'M' => 60, 'H' => '3600'); $value = substr($cadence,2,strlen($cadence)-3); $units = substr($cadence,-1); $sampling = floatval($value)*$scale[$units]; return $sampling; } /** * Get IDs and SPASE URLs of all CDAWEB SPASE-defined datasets ( SPASEql ) * Create an array $CDAWEB[$ID] = SpaseID */ protected function getAllSpaseDatasets() { $this->initStreamContext(); require_once "simple_html_dom.php"; if (file_exists($this->location."/NumericalData.html")) rename($this->location."/NumericalData.html",$this->location."/NumericalData.html.bak"); // if cannot reach CDAWEB Spase Registry use an old file if (!copy(CDAWebConfigClass::$spaseRegistry, $this->location."/NumericalData.html")) { error_log('ERROR'); copy($this->location."/NumericalData.html.bak", $this->location."/NumericalData.html"); } $doc = new DOMDocument('1.0', 'utf-8'); if (!$doc->loadHTMLFile($this->location.'/NumericalData.html')) { error_log("Cannot load ".$this->location.'/NumericalData.html'); } $xpath = new DOMXPath($doc); $ids = $xpath->query("//td[@class='Spase.URL.ProductID']"); foreach ($ids as $id) { $key = $id->nodeValue; $spase = $id->nextSibling->nodeValue; if (!empty($key) && (strlen($spase) > 25)) { $this->CDAWEB[$key] = $spase; } } error_log("SPASE-defined CDAWEB datasets from ".CDAWebConfigClass::$spaseRegistry." : ".count($this->CDAWEB).PHP_EOL,3,log); echo "SPASE-defined CDAWEB datasets : ".count($this->CDAWEB).PHP_EOL; } protected function initDDServerXml($ds, $ins, $obs) { $this->DDserverXml = new DomDocument("1.0"); $this->DDserverXmlName = $this->DDserverDir."/".$ds.".xml"; $rootNode = $this->DDserverXml->createElement('VI'); $this->DDserverXml->appendChild($rootNode); $rootNode->appendChild($this->DDserverXml->createElement("DataBaseID", $this->baseID)); $rootNode->appendChild($this->DDserverXml->createElement("Mission", $obs)); $rootNode->appendChild($this->DDserverXml->createElement("Instrument", $ins)); $rootNode->appendChild($this->DDserverXml->createElement("DataSetID", $ds)); $rootNode->appendChild($this->DDserverXml->createElement("GlobalStart")); $rootNode->appendChild($this->DDserverXml->createElement("GlobalStop")); $rootNode->appendChild($this->DDserverXml->createElement("MinSampling")); // fill value by default - if different - add into param description $rootNode->appendChild($this->DDserverXml->createElement("FillValue", -1.0E31)); } protected function updateDDServerXml($target, $value) { $node = $this->DDserverXml->getElementsByTagName($target); $node->item(0)->nodeValue = $value; } protected function saveDDServerXml() { $this->DDserverXml->save($this->DDserverXmlName); } protected function existsMasterCdf($dsId) { $this->initStreamContext(); $file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf"; $file_headers = @get_headers($file); if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') { $exists = false; error_log("INFO: no master CDF ".$dsId.PHP_EOL,3,log); } else { $exists = true; } return $exists; } protected function getMasterCdf($dsId) { $this->initStreamContext(); $file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf"; $file_headers = @get_headers($file); if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') { return false; } return $file; } public function getData($ds, $start, $stop) { $this->initStreamContext(); $this->openConnection(); $res = $this->loadFromCDAWebWS("datasets/$ds/orig_data/$start,$stop/", array()); if (!$res) { return array(); } if ($res->getElementsByTagName("html")->length > 0) { error_log("ERROR no response for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err); error_log($res->saveXML(),3,err); $this->closeConnection(); return array(); } $fileNames = $res->getElementsByTagName("Name"); $nc_prefix = strlen($ds) > RemoteDataCenterClass::$MAX_VI_NAME_LENGTH ? substr(strtolower($ds),0,RemoteDataCenterClass::$MAX_VI_NAME_LENGTH - 1): strtolower($ds); $files = array(); for ($i = 0; $i < $fileNames->length; $i++) { $fileName = $fileNames->item($i); $url = $fileName->nodeValue; $file_headers = @get_headers($url); if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') { error_log("ERROR 404 Not Found for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err); continue; } $temp = explode('/',$url); $destination = $temp[count($temp)-1]; if (copy($url, $destination)) { $ncFile = $this->convert2nc($destination); if (strlen($ncFile) > RemoteDataCenterClass::$MAX_NAME_LENGTH) { $ncFileTrue = $nc_prefix."_".time().$i.".nc"; rename($ncFile, $ncFileTrue); $files[] = $ncFileTrue; } else { $files[] = $ncFile; } } else { error_log("ERROR cannot copy files : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err); } } $this->closeConnection(); return $files; } protected function convert2nc($file) { system("cdfnew2nc $file"); $ncFile = str_replace(".cdf", ".nc", $file); if (!file_exists($ncFile)) { error_log("ERROR while converting ".$file.PHP_EOL,3,err); return false; } unlink($file); return $ncFile; } public function getDatasetInfo($ds) { if (!$this->existsMasterCdf($ds)) return false; $masterCdf = $this->getMasterCdf($ds); $localCdf = strtolower($ds).".cdf"; if (!copy($masterCdf, $localCdf)) return false; $infoFile = $this->convert2nc($localCdf); return $infoFile; } public function makeProxy() { $this->spase_res = new DomDocument("1.0"); $this->domAmda = new DOMDocument('1.0', 'utf-8'); $this->domAmda->formatOutput = TRUE; $this->domAmda->preserveWhiteSpace = FALSE; $dataRoot = $this->domAmda->createElement('dataRoot'); $dataRoot->setAttribute('xml:id', 'myRemoteData-treeRootNode'); $this->domAmda->appendChild($dataRoot); $this->dataCenter=$this->domAmda->createElement('dataCenter'); $this->setDataCenterAttributes(); $dataRoot->appendChild($this->dataCenter); foreach ($this->obsGroupsIds as $groupId => $obss) { if (in_array($groupId, $this->excludeGroup)) continue; echo $groupId.PHP_EOL; $obsNodes = array(); foreach ($obss as $obs => $inss) { // ! observatory OMNI is taken for observatoriesGroup OMNI only ! if (substr($obs,0,4) == "OMNI" && substr($groupId,0,4) != "OMNI"){ echo 'OMNI '.$obs.' '.$missionId.PHP_EOL; continue; } $insNodes = array(); // Create instrument nodes foreach ($inss as $ins => $dss) { $dsNodes = array(); // Create datasets nodes foreach ($dss as $ds) { $dsId = $this->baseID.":".$ds; if (!$this->domAmda->getElementById($dsId)){ $this->initDDServerXml($ds,$ins,$obs); $dsNode = $this->createDatasetNode($ds); if ($dsNode){ $this->saveDDServerXml(); $dsNodes[] = $dsNode; } } } // foreach ($dss as $ds) if (!empty($dsNodes)) { // last $spase_res : instrument should be the same $insSpaseId = $this->getInstrumentSpase(); if (!$insSpaseId || is_array($insSpaseId)) $insSpaseId = $ins; $obsSpaseId = strtolower($this->getObservatorySpase()); if (!$obsSpaseId ) $obsSpaseId = strtolower($obs); $insId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId).":".$this->param2dd($insSpaseId); if (!($insNode = $this->domAmda->getElementById($insId))){ $insNode = $this->createInstrumentNode($insSpaseId, $obsSpaseId, $groupId); } foreach ($dsNodes as $dsNode){ $insNode->appendChild($dsNode); } $insNodes[] = $insNode; } } // foreach ($inss as $ins => $dss) if (!empty($insNodes)) { $obsId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId); if (!($obsNode = $this->domAmda->getElementById($obsId))){ $obsNode = $this->createObservatoryNode($obsSpaseId, $groupId); } foreach ($insNodes as $insNode){ $obsNode->appendChild($insNode); } $obsNodes[] = $obsNode; } } // foreach ($obss as $obs => $inss) if (!empty($obsNodes)) { $missionId = $this->baseID.":".$this->param2dd($groupId); if (!($missionNode = $this->domAmda->getElementById($missionId))){ $missionNode = $this->createMissionNode($groupId); } foreach ($obsNodes as $obsNode) { // observatory == mission if (strtolower($obsNode->getAttribute('name')) == strtolower($missionNode->getAttribute('name'))) { $insNodes = $obsNode->getElementsByTagName("instrument"); foreach ($insNodes as $insNode) { $missionNode->appendChild($insNode->cloneNode(true)); } } else $missionNode->appendChild($obsNode); } $this->dataCenter->appendChild($missionNode); } } $this->closeConnection(); } protected function createMissionNode($groupId) { $missionId = $this->baseID.":".$this->param2dd($groupId); $missionNode = $this->domAmda->createElement('mission'); $missionNode->setAttribute("xml:id",$missionId); $missionNode->setAttribute("name",$groupId); return $missionNode; } protected function getInstrumentSpase() { $this->insXML = new DomDocument("1.0"); if ($this->spase_res){ $instrument = $this->spase_res->getElementsByTagName('InstrumentID'); if ($instrument->length > 0) { if ($instrument->length == 1) { $this->loadSpaseResource($instrument->item(0)->nodeValue, $this->insXML); return $this->getIdFromSpase($instrument->item(0)->nodeValue); } else { // $insIds = ""; $insIds = array(); for ($i = 0; $i < $instrument->length; $i++ ) { // $insIds .= $this->getIdFromSpase($instrument->item($i)->nodeValue); // if ($i < $instrument->length - 1) // $insIds .= "/"; $insIds[] = $this->getIdFromSpase($instrument->item($i)->nodeValue); } $this->loadSpaseResource($instrument->item($instrument->length - 1)->nodeValue, $this->insXML); return $insIds; } //InstrumentType //spase://SMWG/Observatory/ACE } } return null; } protected function getObservatorySpase() { if ($this->insXML){ $observatory = $this->insXML->getElementsByTagName('ObservatoryID'); if ($observatory->length > 0) { return $this->getIdFromSpase($observatory->item(0)->nodeValue); } } return null; } protected function setDataCenterAttributes(){ $this->dataCenter->setAttribute('xml:id', $this->baseID); // $this->dataCenter->setIdAttribute('xml:id', true); $this->dataCenter->setAttribute('name', $this->baseID); } protected function makeArgumentsList(){} private function loadSpaseResource($resourceID, &$dom) { if (empty($resourceID)) { return FALSE; } $url = str_replace('spase://', 'https://hpde.io/', $resourceID) . '.xml'; curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/xml")); curl_setopt($this->ch, CURLOPT_URL, $url); if (!$dom->loadXML(curl_exec($this->ch))) { error_log('Cannot load Spase Resource: '.$resourceID); return FALSE; } return TRUE; } private function loadFromCDAWebWS($api, $params, $asJSON = FALSE) { if (!$this->dataViewURL) { curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$restUrl."/dataviews"); curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/xml")); $res = new DomDocument(); if (!$res->loadXML(curl_exec($this->ch))) { return FALSE; } if ($res->getElementsByTagName('EndpointAddress')->length == 0) { error_log("Problem connect to ".CDAWebConfigClass::$restUrl."/dataviews", 3, err); return FALSE; } $this->dataViewURL = $res->getElementsByTagName('EndpointAddress')->item(0)->nodeValue; } $url = $this->dataViewURL . '/' . $api; if (!empty($params)) { $params_str = ""; foreach ($params as $key => $value) { if (!empty($params_str)) { $params_str .= "&"; } $params_str .= $key."=".urlencode($value); } $url .= '?'.$params_str; } if ($asJSON) { curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json")); } else { curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/xml")); } curl_setopt($this->ch, CURLOPT_URL, $url); $res = curl_exec($this->ch); if (!$res) { return FALSE; } if ($asJSON) { return json_decode($res); } $dom = new DOMDocument(); if (!$dom->loadXML(curl_exec($this->ch))) { return FALSE; } return $dom; } } ?>