"%20"); return strtr($param,$pairs); } public function init() { error_log("CDAWEB Proxy creation on ".date("Y-m-d").PHP_EOL,3,log); error_log("CDAWEB Proxy creation on ".date("Y-m-d").PHP_EOL,3,err); $this->getAllSpaseDatasets(); } private function openConnection() { $this->ch = curl_init(); curl_setopt($this->ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($this->ch, CURLOPT_TIMEOUT, 60); } private function closeConnection() { curl_close($this->ch); } protected function setDataViewURL() { curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$restUrl."/dataviews"); $this->res = new DomDocument(); $this->res->loadXML(curl_exec($this->ch)); if ($this->res->getElementsByTagName('EndpointAddress')->length == 0) { error_log("Problem connect to ".CDAWebConfigClass::$restUrl."/dataviews", 3, err); exit("Problem connect to ".CDAWebConfigClass::$restUrl."/dataviews".PHP_EOL); } $this->dataViewURL = $this->res->getElementsByTagName('EndpointAddress')->item(0)->nodeValue; } // Using CDAS REST APIs get all CDAWEB dataset descriptions protected function getRemoteTree() { $this->openConnection(); $this->setDataViewURL(); $this->obsGroupsIds = array(); foreach ($this->validInstrumentTypes as $insType) { curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/observatoryGroupsAndInstruments?instrumentType=".$insType); echo $insType.PHP_EOL; $this->res->loadXML(curl_exec($this->ch)); $observatoryGroups = $this->res->getElementsByTagName("ObservatoryGroupInstrumentDescription"); foreach ($observatoryGroups as $obsGroup) { $obsGroupId = $obsGroup->getElementsByTagName('Name')->item(0)->nodeValue; // Exclude some Missions if (in_array ($obsGroupId, $this->excludeGroup )) continue; echo " ".$obsGroupId.PHP_EOL; $observatories = $obsGroup->getElementsByTagName("ObservatoryInstruments"); if ( $observatories->length == 0 ) { echo ' No observatories '.PHP_EOL; if ($obsGroup->getElementsByTagName("InstrumentDescription")->length > 0) { echo ' CHECK INSTRUMENTS '.PHP_EOL; error_log(" CHECK INSTRUMENTS ".$obsGroup.PHP_EOL, 3, err); } continue; } $obsIds = array(); foreach ( $observatories as $obs ) { $obsId = $obs->getElementsByTagName('Name')->item(0)->nodeValue; echo " ".$obsId.PHP_EOL; // // MAG // ACE Magnetic Field Instrument // ACE Magnetic Field Instrument // $instruments = $obs->getElementsByTagName("InstrumentDescription"); $insIds = array(); foreach ( $instruments as $ins ) { $insId = $ins->getElementsByTagName('Name')->item(0)->nodeValue; // Exclude some 'non-AMDA' staff if (in_array ($insId, $this->excludeIns )) continue; $dsIds = $this->getDatasetByObservatoryAndInstrument($obsGroupId, $obsId, $insId); $insIds[$insId] = $dsIds; } $obsIds[$obsId] = $insIds; } $this->obsGroupsIds[$insType][$obsGroupId] = $obsIds; } } } protected function getDatasetByObservatoryAndInstrument($obsGroupId, $obsId, $insId) { $grpEncodedId = $this->html_encode($obsGroupId); $insEncodedId = $this->html_encode($insId); $obsEncodedId = $this->html_encode($obsId); $res = new DomDocument("1.0"); // curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets?observatoryGroup=$grpEncodedId&instrument=$insEncodedId"); curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets?observatory=$obsEncodedId&instrument=$insEncodedId"); $res->loadXML(curl_exec($this->ch)); $datasets = $res->getElementsByTagName('DatasetDescription'); $dsIds = array(); foreach ( $datasets as $ds) { // $obs = $ds->getElementsByTagName('Observatory')->item(0)->nodeValue; // if ($obs != $obsId ) continue; $dsIds[] = $ds->getElementsByTagName('Id')->item(0)->nodeValue; } return $dsIds; } protected function createObservatoryNode($id, $grpId) { $obsNode = $this->domAmda->createElement('observatory'); $obsNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($grpId).":".$this->param2dd($id)); $obsNode->setAttribute("name",$id); return $obsNode; } protected function createInstrumentNode($id, $obsId, $groupId) { $insNode = $this->domAmda->createElement('instrument'); $insNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsId).":".$id); $insNode->setAttribute("name",$id); // $insNode->setAttribute("description",$ins[1]); return $insNode; } protected function createDatasetNode($dsId) { $dsNode = $this->domAmda->createElement('dataset'); curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json")); curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets/?idPattern=".$dsId); $obj = json_decode(curl_exec($this->ch)); $dataSet = $obj->DatasetDescription; $dsNode->setAttribute("xml:id",$this->baseID.":".$dsId); $dsNode->setAttribute("name", $dsId); $startTime = $dataSet[0]->TimeInterval->Start; $endTime = $dataSet[0]->TimeInterval->End; $label = $dataSet[0]->Label; $url = $dataSet[0]->DatasetLink[0]->Url; $notesUrl = $dataSet[0]->Notes; $piAffilation = $dataSet[0]->PiAffiliation; $piName = $dataSet[0]->PiName; $sampling = $this->getDatasetSpaseDescription($dsId); $this->updateDDServerXml("GlobalStart",$startTime); $this->updateDDServerXml("GlobalStop",$endTime); // no general description - strange dataset if ($sampling == -1) { error_log("! No general description : ".$dsId.PHP_EOL,3,err); return null; } if ($sampling < -1) { echo " !!! $dsId $sampling".PHP_EOL; error_log("! No sampling : ".$dsId." : ".$sampling.PHP_EOL,3,err); } else $this->updateDDServerXml("MinSampling",$sampling); $dsNode->setAttribute('spaseUrl',$this->CDAWEB[$dsId]); // $dsNode->setAttribute('masterCdf',$this->existsMasterCdf($dsId)); $dsNode->setAttribute("desc", "$label; $startTime - $endTime"); $dsNode->setAttribute("start", $startTime); $dsNode->setAttribute("stop", $stopTime); $parameterNodes = $this->createParameterNodes($dsId); foreach ( $parameterNodes as $parameterNode) { $dsNode->appendChild($parameterNode); } return $dsNode; } protected function createParameterNodes($dsId) { curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json")); curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets/".$dsId."/variables"); $obj = json_decode(curl_exec($this->ch)); $parameters = $obj->VariableDescription; $paramNodes = array(); foreach ($parameters as $param) { $paramNode = $this->domAmda->createElement('parameter'); $paramNode->setAttribute("xml:id", $this->baseID.":".$dsId.":".$param->Name); $paramNode->setAttribute("name", $param->Name); $paramNodes[] = $paramNode; } return $paramNodes; } protected function getDatasetSpaseDescription($dsID) { // $this->openConnection(); curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/xml")); if ( !array_key_exists($dsID, $this->CDAWEB )) return -1; // no description in SpaseRegistry curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$spaseResolver."id=".$this->CDAWEB[$dsID]); if (!$this->spase_res->loadXML(curl_exec($this->ch))) return -100; $messages = $this->spase_res->getElementsByTagName('Message'); if ($messages->length > 0) { foreach ($messages as $message) echo $message->nodeValue.PHP_EOL; return -2; // no description in www-spase } // $instrument = $this->spase_res->getElementsByTagName('InstrumentID'); // echo " instrument ".$instrument->item(0)->nodeValue.PHP_EOL; $cadence = $this->spase_res->getElementsByTagName('Cadence'); if ($cadence->length == 0) return -3; // no cadence in spase xml $sampling = $this->cadence2sampling($cadence->item(0)->nodeValue); return $sampling; } public function cadence2sampling($cadence) { $scale = array('S' => 1, 'M' => 60, 'H' => '3600'); $value = substr($cadence,2,strlen($cadence)-3); $units = substr($cadence,-1); $sampling = floatval($value)*$scale[$units]; return $sampling; } // Get IDs and SPASE URLs of all CDAWEB SPASE-defined datasets protected function getAllSpaseDatasets() { require_once "simple_html_dom.php"; if (file_exists($this->location."/NumericalData.html")) rename($this->location."/NumericalData.html",$this->location."/NumericalData.html.bak"); // if cannot reach CDAWEB Spase Registry use an old file if (!copy(CDAWebConfigClass::$spaseRegistry, $this->location."/NumericalData.html")) copy($this->location."/NumericalData.html.bak", $this->location."/NumericalData.html"); $html = file_get_html($this->location.'/NumericalData.html'); $ids = $html->find('td[class="Spase.URL.ProductID"]'); foreach ($ids as $id) { $ref = $id->next_sibling()->find('a'); $key = $id->find('a'); $this->CDAWEB[$key[0]->innertext] = $ref[0]->innertext; } error_log("SPASE-defined CDAWEB datasets from ".CDAWebConfigClass::$spaseRegistry." : ".count($this->CDAWEB).PHP_EOL,3,log); echo "SPASE-defined CDAWEB datasets : ".count($this->CDAWEB).PHP_EOL; } protected function initDDServerXml($ds, $ins, $obs) { $this->DDserverXml = new DomDocument("1.0"); $this->DDserverXmlName = $this->DDserverDir."/".$ds.".xml"; $rootNode = $this->DDserverXml->createElement('VI'); $this->DDserverXml->appendChild($rootNode); $rootNode->appendChild($this->DDserverXml->createElement("DataBaseID", $this->baseID)); $rootNode->appendChild($this->DDserverXml->createElement("Mission", $obs)); $rootNode->appendChild($this->DDserverXml->createElement("Instrument", $ins)); $rootNode->appendChild($this->DDserverXml->createElement("DataSetID", $ds)); $rootNode->appendChild($this->DDserverXml->createElement("GlobalStart")); $rootNode->appendChild($this->DDserverXml->createElement("GlobalStop")); $rootNode->appendChild($this->DDserverXml->createElement("MinSampling")); // fill value by default - if different - add into param description $rootNode->appendChild($this->DDserverXml->createElement("FillValue", -1.0E31)); } protected function updateDDServerXml($target, $value) { $node = $this->DDserverXml->getElementsByTagName($target); $node->item(0)->nodeValue = $value; } protected function saveDDServerXml() { $this->DDserverXml->save($this->DDserverXmlName); } protected function existsMasterCdf($dsId) { $file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf"; $file_headers = @get_headers($file); if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') { $exists = false; error_log("INFO: no master CDF ".$dsId.PHP_EOL,3,log); } else { $exists = true; } return $exists; } protected function getMasterCdf($dsId) { $file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf"; $file_headers = @get_headers($file); if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') { return false; } return $file; } public function getData($ds, $start, $stop) { $this->openConnection(); $this->setDataViewURL(); curl_setopt($this->ch, CURLOPT_URL,$this->dataViewURL."/datasets/$ds/orig_data/$start,$stop/"); $res = new DomDocument("1.0"); $res->loadXML(curl_exec($this->ch)); if ($res->getElementsByTagName("html")->length > 0) { error_log("ERROR no response for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err); error_log($res->saveXML(),3,err); exit(); } $fileNames = $res->getElementsByTagName("Name"); $nc_prefix = strlen($ds) > RemoteDataCenterClass::$MAX_VI_NAME_LENGTH ? substr(strtolower($ds),0,RemoteDataCenterClass::$MAX_VI_NAME_LENGTH - 1): strtolower($ds); $files = array(); for ($i = 0; $i < $fileNames->length; $i++) { $fileName = $fileNames->item($i); $url = $fileName->nodeValue; $file_headers = @get_headers($url); if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') { error_log("ERROR 404 Not Found for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err); continue; } $temp = explode('/',$url); $destination = $temp[count($temp)-1]; if (copy($url, $destination)) { $ncFile = $this->convert2nc($destination); if (strlen($ncFile) > RemoteDataCenterClass::$MAX_NAME_LENGTH) { $ncFileTrue = $nc_prefix."_".time().$i.".nc"; rename($ncFile, $ncFileTrue); $files[] = $ncFileTrue; } else { $files[] = $ncFile; } } else { error_log("ERROR cannot copy files : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err); } } $this->closeConnection(); return $files; } protected function convert2nc($file) { system("cdfnew2nc $file"); $ncFile = str_replace(".cdf", ".nc", $file); if (!file_exists($ncFile)) { error_log("ERROR while converting ".$file.PHP_EOL,3,err); return false; } unlink($file); return $ncFile; } public function getDatasetInfo($ds) { $masterCdf = strtolower($ds)."_00000000_v01.cdf"; $localCdf = strtolower($ds).".cdf"; if (!copy(CDAWebConfigClass::$masterUrl."/".$masterCdf, strtolower($ds).".cdf")) return false; $infoFile = $this->convert2nc($localCdf); return $infoFile; } public function makeProxy() { $this->spase_res = new DomDocument("1.0"); $this->domAmda = new DOMDocument('1.0', 'utf-8'); $this->domAmda->formatOutput = TRUE; $this->domAmda->preserveWhiteSpace = FALSE; $dataRoot = $this->domAmda->createElement('dataRoot'); $dataRoot->setAttribute('xml:id', 'myRemoteData-treeRootNode'); $this->domAmda->appendChild($dataRoot); $this->dataCenter=$this->domAmda->createElement('dataCenter'); $this->setDataCenterAttributes(); $dataRoot->appendChild($this->dataCenter); $first = true; foreach ($this->obsGroupsIds as $insType => $groupIds) { // getElementByID doesn't work on newly created Document if (!$first) { $this->domAmda->loadXML($this->domAmda->saveXML()); $this->dataCenter = $this->domAmda->getElementById($this->baseID); } foreach ($groupIds as $groupId => $obss) { $obsNodes = array(); foreach ($obss as $obs => $inss) { // ! observatory OMNI is taken for observatoriesGroup OMNI only ! if (substr($obs,0,4) == "OMNI" && substr($groupId,0,4) != "OMNI"){ echo 'OMNI '.$obs.' '.$missionId.PHP_EOL; continue; } $insNodes = array(); // Create instrument nodes foreach ($inss as $ins => $dss) { $dsNodes = array(); // Create datasets nodes foreach ($dss as $ds) { $dsId = $this->baseID.":".$ds; if (!$this->domAmda->getElementById($dsId)){ $this->initDDServerXml($ds,$ins,$obs); $dsNode = $this->createDatasetNode($ds); if ($dsNode){ $this->saveDDServerXml(); $dsNodes[] = $dsNode; } } } // foreach ($dss as $ds) if ($dsNodes) { // last $spase_res : instrument should be the same $insSpaseId = $this->getInstrumentSpase(); if (!$insSpaseId ) $insSpaseId = $ins; $obsSpaseId = strtolower($this->getObservatorySpase()); if (!$obsSpaseId ) $obsSpaseId = strtolower($obs); $insId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId).":".$this->param2dd($insSpaseId); if (!($insNode = $this->domAmda->getElementById($insId))){ $insNode = $this->createInstrumentNode($insSpaseId, $obsSpaseId, $groupId); } foreach ($dsNodes as $dsNode){ $insNode->appendChild($dsNode); } $insNodes[] = $insNode; } } // foreach ($inss as $ins => $dss) if ($insNodes) { $obsId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId); if (!($obsNode = $this->domAmda->getElementById($obsId))){ $obsNode = $this->createObservatoryNode($obsSpaseId, $groupId); } foreach ($insNodes as $insNode){ $obsNode->appendChild($insNode); } $obsNodes[] = $obsNode; } } // foreach ($obss as $obs => $inss) if ($obsNodes) { $missionId = $this->baseID.":".$this->param2dd($groupId); if (!($missionNode = $this->domAmda->getElementById($missionId))){ $missionNode = $this->createMissionNode($groupId); $this->dataCenter->appendChild($missionNode); } foreach ($obsNodes as $obsNode) { // observatory == mission if (strtolower($obsNode->getAttribute('name')) == strtolower($missionNode->getAttribute('name'))) { $insNodes = $obsNode->getElementsByTagName("instrument"); foreach ($insNodes as $insNode) $missionNode->appendChild($insNode); } else $missionNode->appendChild($obsNode); } } } $first = false; } $this->closeConnection(); } protected function createMissionNode($groupId) { $missionId = $this->baseID.":".$this->param2dd($groupId); $missionNode = $this->domAmda->createElement('mission'); $missionNode->setAttribute("xml:id",$missionId); $missionNode->setAttribute("name",$groupId); return $missionNode; } protected function getInstrumentSpase() { $this->insXML = new DomDocument("1.0"); if ($this->spase_res){ $instrument = $this->spase_res->getElementsByTagName('InstrumentID'); if ($instrument->length > 0) { curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$spaseResolver."id=".$instrument->item(0)->nodeValue); $this->insXML->loadXML(curl_exec($this->ch)); return $this->getIdFromSpase($instrument->item(0)->nodeValue); //InstrumentType //spase://SMWG/Observatory/ACE } } return null; } protected function getObservatorySpase() { if ($this->insXML){ $observatory = $this->insXML->getElementsByTagName('ObservatoryID'); if ($observatory->length > 0) { return $this->getIdFromSpase($observatory->item(0)->nodeValue); } } return null; } protected function setDataCenterAttributes(){ $this->dataCenter->setAttribute('xml:id', $this->baseID); // $this->dataCenter->setIdAttribute('xml:id', true); $this->dataCenter->setAttribute('name', $this->baseID); } protected function makeArgumentsList(){} } ?>