"%20"); return strtr($param,$pairs); } public function init() { $this->getAllSpaseDatasets(); } private function openConnection() { $this->ch = curl_init(); curl_setopt($this->ch, CURLOPT_RETURNTRANSFER, 1); curl_setopt($this->ch, CURLOPT_TIMEOUT, 60); } private function closeConnection() { curl_close($this->ch); } public function initProxy() { $this->dir = getenv("DDBASE")."/../INFO"; define('RemoteData', $this->dir."/bases/"); // if (!is_dir(RemoteData)) mkdir(RemoteData, 0755, true); if (!is_dir($this->dir."/bases/".$this->baseID)) mkdir($this->dir."/bases/".$this->baseID, 0755, true); $this->DDserverDir = $this->dir."/DDServer/".$this->baseID; if (!is_dir($this->DDserverDir)) mkdir($this->DDserverDir, 0755, true); $this->init(); $this->getRemoteTree(); } protected function setDataViewURL() { curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$restUrl."/dataviews"); $this->res = new DomDocument(); $this->res->loadXML(curl_exec($this->ch)); $this->dataViewURL = $this->res->getElementsByTagName('EndpointAddress')->item(0)->nodeValue; } protected function getRemoteTree() { $this->openConnection(); $this->setDataViewURL(); $this->obsGroupsIds = array(); foreach ( $this->validInstrumentTypes as $insType) { curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/observatoryGroupsAndInstruments?instrumentType=".$insType); echo $insType.PHP_EOL; $this->res->loadXML(curl_exec($this->ch)); // $this->res->save($insType.".xml"); $observatoryGroups = $this->res->getElementsByTagName("ObservatoryGroupInstrumentDescription"); foreach ( $observatoryGroups as $obsGroup ) { $obsGroupId = $obsGroup->getElementsByTagName('Name')->item(0)->nodeValue; // Exclude some Missions if (in_array ($obsGroupId, $this->excludeGroup )) continue; echo " ".$obsGroupId.PHP_EOL; $observatories = $obsGroup->getElementsByTagName("ObservatoryInstruments"); if ( $observatories->length == 0 ) { echo ' No observatories '.PHP_EOL; if ($obsGroup->getElementsByTagName("InstrumentDescription")->length > 0) echo ' CHECK INSTRUMENTS '.PHP_EOL; continue; } $obsIds = array(); foreach ( $observatories as $obs ) { $obsId = $obs->getElementsByTagName('Name')->item(0)->nodeValue; echo " ".$obsId.PHP_EOL; // // MAG // ACE Magnetic Field Instrument // ACE Magnetic Field Instrument // $instruments = $obs->getElementsByTagName("InstrumentDescription"); $insIds = array(); foreach ( $instruments as $ins ) { $insId = $ins->getElementsByTagName('Name')->item(0)->nodeValue; // Exclude some 'non-AMDA' staff if (in_array ($insId, $this->excludeIns )) continue; $dsIds = $this->getDatasetByObservatoryAndInstrument($obsGroupId, $obsId, $insId); $insIds[$insId] = $dsIds; } $obsIds[$obsId] = $insIds; } $this->obsGroupsIds[$obsGroupId][$insType] = $obsIds; } } // print_r($this->obsGroupsIds); // $this->closeConnection(); } protected function getDatasetByObservatoryAndInstrument($obsGroupId, $obsId, $insId) { // echo "In getDataset ".$obsId." ".$insId.PHP_EOL; $grpEncodedId = $this->html_encode($obsGroupId); $insEncodedId = $this->html_encode($insId); $res = new DomDocument("1.0"); curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets?observatoryGroup=$grpEncodedId&instrument=$insEncodedId"); $res->loadXML(curl_exec($this->ch)); // $this->res->save('dataset.xml'); // exit(); $datasets = $res->getElementsByTagName('DatasetDescription'); $dsIds = array(); foreach ( $datasets as $ds) { $obs = $ds->getElementsByTagName('Observatory')->item(0)->nodeValue; if ($obs != $obsId ) continue; $dsIds[] = $ds->getElementsByTagName('Id')->item(0)->nodeValue; } return $dsIds; } protected function createMissionNodes() { $missionNodes = array(); $missionIds = array(); // unique values // $this->obsGroupsIds = array_map("unserialize", array_unique(array_map("serialize", $this->obsGroupsIds))); foreach ($this->obsGroupsIds as $groupId => $insTypes) { // echo ' GroupID '.$groupId.PHP_EOL; $missionId = $this->baseID.":".$this->param2dd($groupId); if (!in_array($missionId, $missionIds)) { $missionNode = $this->domAmda->createElement('mission'); $missionNode->setAttribute("xml:id",$missionId); $missionNode->setAttribute("name",$groupId); $missionIds[] = $missionId; $newNode = true; } else { $key = array_search($missionId, $missionIds); $missionNode = $missionNodes[$key]; $newNode = false; } foreach ($insTypes as $insType => $obss) { // echo $insType.PHP_EOL; foreach ( $obss as $obs => $inss) { // echo $obs.PHP_EOL; // OMNI ONLY ONCE if (substr($obs,0,4) == "OMNI" && substr($missionId,0,4) != "OMNI") { //echo ' '.$obs.' '.$missionId.PHP_EOL; continue; } $obsId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obs); $obsNodes = $missionNode->getElementsByTagName('observatory'); $newObsNode = true; foreach ($obsNodes as $node) { $id = $node->getAttribute("xml:id"); if ($id == $obsId) { $obsNode = $node; $newObsNode = false; break; } } if ( $newObsNode ) { $obsNode = $this->createObservatoryNode($obs, $groupId); } foreach ( $inss as $ins => $dss) { $insId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obs).":".$this->param2dd($ins); $insNodes = $obsNode->getElementsByTagName('instrument'); $newInsNode = true; foreach ($insNodes as $node) { $id = $node->getAttribute("xml:id"); if ($id == $insId) { $insNode = $node; $newInsNode = false; break; } } if ( $newInsNode ) $insNode = $this->createInstrumentNode($ins, $obs, $groupId); foreach ($dss as $ds) { $dsId = $this->baseID.":".$ds; $dsNodes = $insNode->getElementsByTagName('dataset'); $newDsNode = true; foreach ($dsNodes as $node) { $id = $node->getAttribute("xml:id"); if ($id == $dsId) { $dsNode = $node; $newDsNode = false; break; } } if ($newDsNode) { $this->initDDServerXml($ds,$ins,$obs); $dsNode = $this->createDatasetNode($ds); if ($dsNode) { $insNode->appendChild($dsNode); $this->saveDDServerXml(); } } } if ($newInsNode && $insNode->hasChildNodes()) $obsNode->appendChild($insNode); } if ($newObsNode && $obsNode->hasChildNodes()) { $missionNode->appendChild($obsNode); } } } if ($newNode) $missionNodes[] = $missionNode; } return $missionNodes; } protected function createObservatoryNode($id, $grpId) { $insNode = $this->domAmda->createElement('observatory'); $insNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($grpId).":".$this->param2dd($id)); $insNode->setAttribute("name",$id); // $insNode->setAttribute("description",$ins[1]); return $insNode; } protected function createInstrumentNode($id, $obsId, $groupId) { $insNode = $this->domAmda->createElement('instrument'); $insNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsId).":".$id); $insNode->setAttribute("name",$id); // $insNode->setAttribute("description",$ins[1]); return $insNode; } protected function createDatasetNode($dsId) { $dsNode = $this->domAmda->createElement('dataset'); curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json")); curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets/?idPattern=".$dsId); $obj = json_decode(curl_exec($this->ch)); $dataSet = $obj->DatasetDescription; $dsNode->setAttribute("xml:id",$this->baseID.":".$dsId); $dsNode->setAttribute("name", $dsId); $startTime = $dataSet[0]->TimeInterval->Start; $endTime = $dataSet[0]->TimeInterval->End; $label = $dataSet[0]->Label; $url = $dataSet[0]->DatasetLink[0]->Url; $notesUrl = $dataSet[0]->Notes; $piAffilation = $dataSet[0]->PiAffiliation; $piName = $dataSet[0]->PiName; $sampling = $this->getDatasetSpaseDescription($dsId); $this->updateDDServerXml("GlobalStart",$startTime); $this->updateDDServerXml("GlobalStop",$endTime); // no general description - strange dataset if ($sampling == -1) return null; if ($sampling < -1) echo " !!! $dsId $sampling".PHP_EOL; else $this->updateDDServerXml("MinSampling",$sampling); $dsNode->setAttribute('spaseUrl',$this->CDAWEB[$dsId]); // $dsNode->setAttribute('masterCdf',$this->existsMasterCdf($dsId)); $parameterNodes = $this->createParameterNodes($dsId); foreach ( $parameterNodes as $parameterNode) { $dsNode->appendChild($parameterNode); } return $dsNode; } protected function createParameterNodes($dsId) { curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json")); curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets/".$dsId."/variables"); $obj = json_decode(curl_exec($this->ch)); $parameters = $obj->VariableDescription; $paramNodes = array(); foreach ($parameters as $param) { $paramNode = $this->domAmda->createElement('parameter'); $paramNode->setAttribute("xml:id", $this->baseID.":".$dsId.":".$param->Name); $paramNode->setAttribute("name", $param->Name); $paramNodes[] = $paramNode; } return $paramNodes; } protected function getDatasetSpaseDescription($dsID) { // $this->openConnection(); curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/xml")); if ( !array_key_exists($dsID, $this->CDAWEB )) return -1; // no description in SpaseRegistry curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$spaseResolver."id=".$this->CDAWEB[$dsID]); $this->res->loadXML(curl_exec($this->ch)); //TODO errors $messages = $this->res->getElementsByTagName('Message'); if ($messages->length > 0) { foreach ($messages as $message) echo $message->nodeValue.PHP_EOL; return -2; // no description in www-spase } $instrument = $this->res->getElementsByTagName('InstrumentID'); echo " instrument ".$instrument->item(0)->nodeValue.PHP_EOL; $cadence = $this->res->getElementsByTagName('Cadence'); if ( $cadence->length == 0) return -3; // no cadence in spase xml $sampling = $this->cadence2sampling($cadence->item(0)->nodeValue); return $sampling; } public function cadence2sampling($cadence) { $scale = array('S' => 1, 'M' => 60, 'H' => '3600'); $value = substr($cadence,2,strlen($cadence)-3); $units = substr($cadence,-1); $sampling = (int)$value*$scale[$units]; return $sampling; } protected function getAllSpaseDatasets() { require_once "simple_html_dom.php"; if (file_exists("NumericalData.html")) rename("NumericalData.html","NumericalData.html.bak"); // exec("wget -O NumericalData.html ".CDAWebConfigClass::$spaseRegistry); //TODO errors copy(CDAWebConfigClass::$spaseRegistry, "NumericalData.html"); $html = file_get_html('NumericalData.html'); $ids = $html->find('td[class="Spase.URL.ProductID"]'); foreach ($ids as $id) { $ref = $id->next_sibling()->find('a'); $key = $id->find('a'); $this->CDAWEB[$key[0]->innertext] = $ref[0]->innertext; } echo count($this->CDAWEB).PHP_EOL; } protected function initDDServerXml($ds, $ins, $obs) { $this->DDserverXml = new DomDocument("1.0"); $this->DDserverXmlName = $this->DDserverDir."/".$ds.".xml"; $rootNode = $this->DDserverXml->createElement('VI'); $this->DDserverXml->appendChild($rootNode); $rootNode->appendChild($this->DDserverXml->createElement("DataBaseID", $this->baseID)); $rootNode->appendChild($this->DDserverXml->createElement("Mission", $obs)); $rootNode->appendChild($this->DDserverXml->createElement("Instrument", $ins)); $rootNode->appendChild($this->DDserverXml->createElement("DataSetID", $ds)); $rootNode->appendChild($this->DDserverXml->createElement("GlobalStart")); $rootNode->appendChild($this->DDserverXml->createElement("GlobalStop")); $rootNode->appendChild($this->DDserverXml->createElement("MinSampling")); } protected function updateDDServerXml($target, $value) { $node = $this->DDserverXml->getElementsByTagName($target); $node->item(0)->nodeValue = $value; } protected function saveDDServerXml() { $this->DDserverXml->save($this->DDserverXmlName); } protected function existsMasterCdf($dsId) { $file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf"; $file_headers = @get_headers($file); if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') { $exists = false; // echo " NO master cdf ".$dsId.PHP_EOL; } else { $exists = true; } return $exists; } protected function getMasterCdf($dsId) { $file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf"; $file_headers = @get_headers($file); if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') { return false; } return $file; } public function getData($ds, $start, $stop) { $this->openConnection(); $this->setDataViewURL(); curl_setopt($this->ch, CURLOPT_URL,$this->dataViewURL."/datasets/$ds/orig_data/$start,$stop/"); $res = new DomDocument("1.0"); $res->loadXML(curl_exec($this->ch)); //TODO errors if ($res->getElementsByTagName("html")->length > 0) { echo $res->saveXML(); exit(); } $fileNames = $res->getElementsByTagName("Name"); $nc_prefix = strlen($ds) > RemoteDataCenterClass::$MAX_VI_NAME_LENGTH ? substr(strtolower($ds),0,RemoteDataCenterClass::$MAX_VI_NAME_LENGTH - 1): strtolower($ds); $files = array(); for ($i = 0; $i < $fileNames->length; $i++) { $fileName = $fileNames->item($i); $url = $fileName->nodeValue; $file_headers = @get_headers($url); if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') { //TODO errors continue; } $temp = explode('/',$url); $destination = $temp[count($temp)-1]; if (copy($url, $destination)) { $ncFile = $this->convert2nc($destination); if (strlen($ncFile) > RemoteDataCenterClass::$MAX_NAME_LENGTH) { $ncFileTrue = $nc_prefix."_".time().$i.".nc"; rename($ncFile, $ncFileTrue); $files[] = $ncFileTrue; } else { $files[] = $ncFile; } } else { //TODO errors } } $this->closeConnection(); return $files; } protected function convert2nc($file) { //TODO errors system("cdf2nc $file"); $ncFile = str_replace(".cdf", ".nc", $file); if (!file_exists($ncFile)) return false; unlink($file); return $ncFile; } public function getDatasetInfo($ds) { $masterCdf = strtolower($ds)."_00000000_v01.cdf"; $localCdf = strtolower($ds).".cdf"; if (!copy(CDAWebConfigClass::$masterUrl."/".$masterCdf, strtolower($ds).".cdf")) return false; $infoFile = $this->convert2nc($localCdf); return $infoFile; } protected function setDataCenterAttributes(){} protected function makeArgumentsList(){} } ?>