"%20"); return strtr($param,$pairs); } public function init() { $this->getAllSpaseDatasets(); } private function openConnection() { $this->ch = curl_init(); curl_setopt($this->ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($this->ch, CURLOPT_TIMEOUT, 60); } private function closeConnection() { curl_close($this->ch); } protected function setDataViewURL() { curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$restUrl."/dataviews"); $this->res = new DomDocument(); $this->res->loadXML(curl_exec($this->ch)); $this->dataViewURL = $this->res->getElementsByTagName('EndpointAddress')->item(0)->nodeValue; } // Using CDAS REST APIs get all CDAWEB dataset descriptions protected function getRemoteTree() { $this->openConnection(); $this->setDataViewURL(); $this->obsGroupsIds = array(); foreach ( $this->validInstrumentTypes as $insType) { curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/observatoryGroupsAndInstruments?instrumentType=".$insType); echo $insType.PHP_EOL; $this->res->loadXML(curl_exec($this->ch)); // $this->res->save($insType.".xml"); $observatoryGroups = $this->res->getElementsByTagName("ObservatoryGroupInstrumentDescription"); foreach ( $observatoryGroups as $obsGroup ) { $obsGroupId = $obsGroup->getElementsByTagName('Name')->item(0)->nodeValue; // Exclude some Missions if (in_array ($obsGroupId, $this->excludeGroup )) continue; echo " ".$obsGroupId.PHP_EOL; $observatories = $obsGroup->getElementsByTagName("ObservatoryInstruments"); if ( $observatories->length == 0 ) { echo ' No observatories '.PHP_EOL; if ($obsGroup->getElementsByTagName("InstrumentDescription")->length > 0) echo ' CHECK INSTRUMENTS '.PHP_EOL; continue; } $obsIds = array(); foreach ( $observatories as $obs ) { $obsId = $obs->getElementsByTagName('Name')->item(0)->nodeValue; echo " ".$obsId.PHP_EOL; // // MAG // ACE Magnetic Field Instrument // ACE Magnetic Field Instrument // $instruments = $obs->getElementsByTagName("InstrumentDescription"); $insIds = array(); foreach ( $instruments as $ins ) { $insId = $ins->getElementsByTagName('Name')->item(0)->nodeValue; // Exclude some 'non-AMDA' staff if (in_array ($insId, $this->excludeIns )) continue; $dsIds = $this->getDatasetByObservatoryAndInstrument($obsGroupId, $obsId, $insId); $insIds[$insId] = $dsIds; } $obsIds[$obsId] = $insIds; } $this->obsGroupsIds[$insType][$obsGroupId] = $obsIds; } } // print_r($this->obsGroupsIds); // $this->closeConnection(); } protected function getDatasetByObservatoryAndInstrument($obsGroupId, $obsId, $insId) { // echo "In getDataset ".$obsId." ".$insId.PHP_EOL; $grpEncodedId = $this->html_encode($obsGroupId); $insEncodedId = $this->html_encode($insId); $res = new DomDocument("1.0"); curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets?observatoryGroup=$grpEncodedId&instrument=$insEncodedId"); $res->loadXML(curl_exec($this->ch)); // $this->res->save('dataset.xml'); // exit(); $datasets = $res->getElementsByTagName('DatasetDescription'); $dsIds = array(); foreach ( $datasets as $ds) { $obs = $ds->getElementsByTagName('Observatory')->item(0)->nodeValue; if ($obs != $obsId ) continue; $dsIds[] = $ds->getElementsByTagName('Id')->item(0)->nodeValue; } return $dsIds; } protected function createMissionNodes() { $missionNodes = array(); $missionIds = array(); // unique values // $this->obsGroupsIds = array_map("unserialize", array_unique(array_map("serialize", $this->obsGroupsIds))); foreach ($this->obsGroupsIds as $groupId => $insTypes) { // echo ' GroupID '.$groupId.PHP_EOL; $missionId = $this->baseID.":".$this->param2dd($groupId); if (!in_array($missionId, $missionIds)) { $missionNode = $this->domAmda->createElement('mission'); $missionNode->setAttribute("xml:id",$missionId); $missionNode->setAttribute("name",$groupId); $missionIds[] = $missionId; $newNode = true; } else { $key = array_search($missionId, $missionIds); $missionNode = $missionNodes[$key]; $newNode = false; } foreach ($insTypes as $insType => $obss){ // echo $insType.PHP_EOL; foreach ( $obss as $obs => $inss){ // echo $obs.PHP_EOL; // OMNI ONLY ONCE if (substr($obs,0,4) == "OMNI" && substr($missionId,0,4) != "OMNI"){ //echo ' '.$obs.' '.$missionId.PHP_EOL; continue; } if (file_exists(RemoteData."/".CDAWebConfigClass::$mapXML)) { $obsAMDA = $this->alias(RemoteData."/".CDAWebConfigClass::$mapXML, $obs); } else { $obsAMDA = $obs; } $obsId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsAMDA); $obsNodes = $missionNode->getElementsByTagName('observatory'); $newObsNode = true; foreach ($obsNodes as $node){ $id = $node->getAttribute("xml:id"); if ($id == $obsId) { $obsNode = $node; $newObsNode = false; break; } } if ( $newObsNode ){ $obsNode = $this->createObservatoryNode($obs, $groupId); } foreach ($inss as $ins => $dss){ $insId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obs).":".$this->param2dd($ins); $insNodes = $obsNode->getElementsByTagName('instrument'); $newInsNode = true; foreach ($insNodes as $node){ $id = $node->getAttribute("xml:id"); if ($id == $insId) { $insNode = $node; $newInsNode = false; break; } } if ( $newInsNode ) $insNode = $this->createInstrumentNode($ins, $obs, $groupId); foreach ($dss as $ds){ $dsId = $this->baseID.":".$ds; $dsNodes = $insNode->getElementsByTagName('dataset'); $newDsNode = true; foreach ($dsNodes as $node){ $id = $node->getAttribute("xml:id"); if ($id == $dsId){ $dsNode = $node; $newDsNode = false; break; } } if ($newDsNode) { $this->initDDServerXml($ds,$ins,$obs); $dsNode = $this->createDatasetNode($ds); if ($dsNode){ $insNode->appendChild($dsNode); $this->saveDDServerXml(); } } } if ($newInsNode && $insNode->hasChildNodes()) $obsNode->appendChild($insNode); } if ($newObsNode && $obsNode->hasChildNodes()){ $missionNode->appendChild($obsNode); } } } if ($newNode) $missionNodes[] = $missionNode; } return $missionNodes; } protected function createObservatoryNode($id, $grpId) { $obsNode = $this->domAmda->createElement('observatory'); $obsNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($grpId).":".$this->param2dd($id)); $obsNode->setAttribute("name",$id); return $obsNode; } protected function createInstrumentNode($id, $obsId, $groupId) { $insNode = $this->domAmda->createElement('instrument'); $insNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsId).":".$id); $insNode->setAttribute("name",$id); // $insNode->setAttribute("description",$ins[1]); return $insNode; } protected function createDatasetNode($dsId) { $dsNode = $this->domAmda->createElement('dataset'); curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json")); curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets/?idPattern=".$dsId); $obj = json_decode(curl_exec($this->ch)); $dataSet = $obj->DatasetDescription; $dsNode->setAttribute("xml:id",$this->baseID.":".$dsId); $dsNode->setAttribute("name", $dsId); $startTime = $dataSet[0]->TimeInterval->Start; $endTime = $dataSet[0]->TimeInterval->End; $label = $dataSet[0]->Label; $url = $dataSet[0]->DatasetLink[0]->Url; $notesUrl = $dataSet[0]->Notes; $piAffilation = $dataSet[0]->PiAffiliation; $piName = $dataSet[0]->PiName; $sampling = $this->getDatasetSpaseDescription($dsId); $this->updateDDServerXml("GlobalStart",$startTime); $this->updateDDServerXml("GlobalStop",$endTime); // no general description - strange dataset if ($sampling == -1) return null; if ($sampling < -1) echo " !!! $dsId $sampling".PHP_EOL; else $this->updateDDServerXml("MinSampling",$sampling); $dsNode->setAttribute('spaseUrl',$this->CDAWEB[$dsId]); // $dsNode->setAttribute('masterCdf',$this->existsMasterCdf($dsId)); $dsNode->setAttribute("desc", "$label; $startTime - $endTime"); $dsNode->setAttribute("start", $startTime); $dsNode->setAttribute("stop", $stopTime); $parameterNodes = $this->createParameterNodes($dsId); foreach ( $parameterNodes as $parameterNode) { $dsNode->appendChild($parameterNode); } return $dsNode; } protected function createParameterNodes($dsId) { curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json")); curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets/".$dsId."/variables"); $obj = json_decode(curl_exec($this->ch)); $parameters = $obj->VariableDescription; $paramNodes = array(); foreach ($parameters as $param) { $paramNode = $this->domAmda->createElement('parameter'); $paramNode->setAttribute("xml:id", $this->baseID.":".$dsId.":".$param->Name); $paramNode->setAttribute("name", $param->Name); $paramNodes[] = $paramNode; } return $paramNodes; } protected function getDatasetSpaseDescription($dsID) { // $this->openConnection(); curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/xml")); if ( !array_key_exists($dsID, $this->CDAWEB )) return -1; // no description in SpaseRegistry curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$spaseResolver."id=".$this->CDAWEB[$dsID]); if (!$this->spase_res->loadXML(curl_exec($this->ch))) return -100; $messages = $this->spase_res->getElementsByTagName('Message'); if ($messages->length > 0) { foreach ($messages as $message) echo $message->nodeValue.PHP_EOL; return -2; // no description in www-spase } // $instrument = $this->spase_res->getElementsByTagName('InstrumentID'); // echo " instrument ".$instrument->item(0)->nodeValue.PHP_EOL; $cadence = $this->spase_res->getElementsByTagName('Cadence'); if ($cadence->length == 0) return -3; // no cadence in spase xml $sampling = $this->cadence2sampling($cadence->item(0)->nodeValue); return $sampling; } public function cadence2sampling($cadence) { $scale = array('S' => 1, 'M' => 60, 'H' => '3600'); $value = substr($cadence,2,strlen($cadence)-3); $units = substr($cadence,-1); $sampling = floatval($value)*$scale[$units]; return $sampling; } // Get IDs and SPASE URLs of all CDAWEB SPASE-defined datasets protected function getAllSpaseDatasets() { require_once "simple_html_dom.php"; if (file_exists("NumericalData.html")) rename("NumericalData.html","NumericalData.html.bak"); // if cannot reach CDAWEB Spase Registry use an old file if (!copy(CDAWebConfigClass::$spaseRegistry, "NumericalData.html")) copy("NumericalData.html.bak", "NumericalData.html"); $html = file_get_html('NumericalData.html'); $ids = $html->find('td[class="Spase.URL.ProductID"]'); foreach ($ids as $id) { $ref = $id->next_sibling()->find('a'); $key = $id->find('a'); $this->CDAWEB[$key[0]->innertext] = $ref[0]->innertext; } echo "SPASE-defined CDAWEB datasets : ".count($this->CDAWEB).PHP_EOL; } protected function initDDServerXml($ds, $ins, $obs) { $this->DDserverXml = new DomDocument("1.0"); $this->DDserverXmlName = $this->DDserverDir."/".$ds.".xml"; $rootNode = $this->DDserverXml->createElement('VI'); $this->DDserverXml->appendChild($rootNode); $rootNode->appendChild($this->DDserverXml->createElement("DataBaseID", $this->baseID)); $rootNode->appendChild($this->DDserverXml->createElement("Mission", $obs)); $rootNode->appendChild($this->DDserverXml->createElement("Instrument", $ins)); $rootNode->appendChild($this->DDserverXml->createElement("DataSetID", $ds)); $rootNode->appendChild($this->DDserverXml->createElement("GlobalStart")); $rootNode->appendChild($this->DDserverXml->createElement("GlobalStop")); $rootNode->appendChild($this->DDserverXml->createElement("MinSampling")); // fill value by default - if different - add into param description $rootNode->appendChild($this->DDserverXml->createElement("FillValue"), -1.0E31); } protected function updateDDServerXml($target, $value) { $node = $this->DDserverXml->getElementsByTagName($target); $node->item(0)->nodeValue = $value; } protected function saveDDServerXml() { $this->DDserverXml->save($this->DDserverXmlName); } protected function existsMasterCdf($dsId) { $file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf"; $file_headers = @get_headers($file); if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') { $exists = false; // echo " NO master cdf ".$dsId.PHP_EOL; } else { $exists = true; } return $exists; } protected function getMasterCdf($dsId) { $file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf"; $file_headers = @get_headers($file); if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') { return false; } return $file; } public function getData($ds, $start, $stop) { $this->openConnection(); $this->setDataViewURL(); curl_setopt($this->ch, CURLOPT_URL,$this->dataViewURL."/datasets/$ds/orig_data/$start,$stop/"); $res = new DomDocument("1.0"); $res->loadXML(curl_exec($this->ch)); //TODO errors if ($res->getElementsByTagName("html")->length > 0) { echo $res->saveXML(); exit(); } $fileNames = $res->getElementsByTagName("Name"); $nc_prefix = strlen($ds) > RemoteDataCenterClass::$MAX_VI_NAME_LENGTH ? substr(strtolower($ds),0,RemoteDataCenterClass::$MAX_VI_NAME_LENGTH - 1): strtolower($ds); $files = array(); for ($i = 0; $i < $fileNames->length; $i++) { $fileName = $fileNames->item($i); $url = $fileName->nodeValue; $file_headers = @get_headers($url); if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') { //TODO errors continue; } $temp = explode('/',$url); $destination = $temp[count($temp)-1]; if (copy($url, $destination)) { $ncFile = $this->convert2nc($destination); if (strlen($ncFile) > RemoteDataCenterClass::$MAX_NAME_LENGTH) { $ncFileTrue = $nc_prefix."_".time().$i.".nc"; rename($ncFile, $ncFileTrue); $files[] = $ncFileTrue; } else { $files[] = $ncFile; } } else { //TODO errors } } $this->closeConnection(); return $files; } protected function convert2nc($file) { //TODO errors system("cdfnew2nc $file"); $ncFile = str_replace(".cdf", ".nc", $file); if (!file_exists($ncFile)) return false; unlink($file); return $ncFile; } public function getDatasetInfo($ds) { $masterCdf = strtolower($ds)."_00000000_v01.cdf"; $localCdf = strtolower($ds).".cdf"; if (!copy(CDAWebConfigClass::$masterUrl."/".$masterCdf, strtolower($ds).".cdf")) return false; $infoFile = $this->convert2nc($localCdf); return $infoFile; } public function makeProxy() { $this->spase_res = new DomDocument("1.0"); $this->domAmda = new DOMDocument('1.0', 'utf-8'); $this->domAmda->formatOutput = TRUE; $this->domAmda->preserveWhiteSpace = FALSE; $dataRoot = $this->domAmda->createElement('dataRoot'); $dataRoot->setAttribute('xml:id', 'myRemoteData-treeRootNode'); $this->domAmda->appendChild($dataRoot); $this->dataCenter=$this->domAmda->createElement('dataCenter'); $this->dataCenter->setAttribute('xml:id', $this->baseID); // $this->dataCenter->setIdAttribute('xml:id', true); $this->dataCenter->setAttribute('name', $this->baseID); $dataRoot->appendChild($this->dataCenter); $this->setDataCenterAttributes(); $first = true; foreach ($this->obsGroupsIds as $insType => $groupIds){ // getElementByID doesn't work on newly created Document if (!$first) { $this->domAmda->loadXML($this->domAmda->saveXML()); $this->dataCenter = $this->domAmda->getElementById($this->baseID); } foreach ($groupIds as $groupId => $obss){ $obsNodes = array(); foreach ($obss as $obs => $inss) { if (substr($obs,0,4) == "OMNI" && substr($groupId,0,4) != "OMNI"){ echo 'OMNI '.$obs.' '.$missionId.PHP_EOL; continue; } $insNodes = array(); // Create instrument nodes foreach ($inss as $ins => $dss){ $dsNodes = array(); // Create datasets nodes foreach ($dss as $ds){ $dsId = $this->baseID.":".$ds; if (!$this->domAmda->getElementById($dsId)){ $this->initDDServerXml($ds,$ins,$obs); $dsNode = $this->createDatasetNode($ds); if ($dsNode){ $this->saveDDServerXml(); $dsNodes[] = $dsNode; } } } // foreach ($dss as $ds) if ($dsNodes){ // last $spase_res : instrument should be the same $insSpaseId = $this->getInstrumentSpase(); if (!$insSpaseId ) $insSpaseId = $ins; $obsSpaseId = $this->getObservatorySpase(); if (!$obsSpaseId ) $obsSpaseId = $obs; $insId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId).":".$this->param2dd($insSpaseId); if (!($insNode = $this->domAmda->getElementById($insId))){ $insNode = $this->createInstrumentNode($insSpaseId, $obsSpaseId, $groupId); $insNodes[] = $insNode; } foreach ($dsNodes as $dsNode){ $insNode->appendChild($dsNode); } } } // foreach ($inss as $ins => $dss) if ($insNodes){ $obsId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId); if (!($obsNode = $this->domAmda->getElementById($obsId))){ $obsNode = $this->createObservatoryNode($obsSpaseId, $groupId); // if (!$first){ // echo "NEW OBS ".$obsId.PHP_EOL; // } $obsNodes[] = $obsNode; } foreach ($insNodes as $insNode){ $obsNode->appendChild($insNode); } } } // foreach ($obss as $obs => $inss) if ($obsNodes){ $missionId = $this->baseID.":".$this->param2dd($groupId); if (!($missionNode = $this->domAmda->getElementById($missionId))){ $missionNode = $this->createMissionNode($groupId); $this->dataCenter->appendChild($missionNode); } foreach ($obsNodes as $obsNode){ // observatory == mission if ($obsNode->getAttribute('name') == $missionNode->getAttribute('name')) { $insNodes = $obsNode->getElementsByTagName("instrument"); foreach ($insNodes as $insNode) $missionNode->appendChild($insNode); } else $missionNode->appendChild($obsNode); } } } $first = false; } } protected function createMissionNode($groupId){ $missionId = $this->baseID.":".$this->param2dd($groupId); $missionNode = $this->domAmda->createElement('mission'); $missionNode->setAttribute("xml:id",$missionId); $missionNode->setAttribute("name",$groupId); return $missionNode; } protected function getInstrumentSpase(){ $this->insXML = new DomDocument("1.0"); if ($this->spase_res){ $instrument = $this->spase_res->getElementsByTagName('InstrumentID'); if ($instrument->length > 0) { curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$spaseResolver."id=".$instrument->item(0)->nodeValue); $this->insXML->loadXML(curl_exec($this->ch)); return $this->getIdFromSpase($instrument->item(0)->nodeValue); //InstrumentType //spase://SMWG/Observatory/ACE } } return null; } protected function getObservatorySpase(){ if ($this->insXML){ $observatory = $this->insXML->getElementsByTagName('ObservatoryID'); if ($observatory->length > 0) { return $this->getIdFromSpase($observatory->item(0)->nodeValue); } } return null; } protected function setDataCenterAttributes(){} protected function makeArgumentsList(){} } ?>