<?php
/**
 * @class CDAWEB 
 * @brief  CDAWeb 
 * @details
 */
class CDAWEB extends RemoteDataCenterClass
{
	private $ch, $res, $dataViewUR; 
	private $obsGroupsIds;
	
	private $observatoryGroups, $instrumentTypes;
	private $CDAWEB = array(), $masterCDF = array();
	
	// https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/instrumentTypes
	private $validInstrumentTypes = [ "Activity%20Indices", "Electric%20Fields%20(space)","Gamma%20and%20X-Rays",
                                     "Magnetic%20Fields%20(space)", "Particles%20(space)", "Plasma%20and%20Solar%20Wind", 
                                     "Radio%20and%20Plasma%20Waves%20(space)" ];                                     
//   not used : $validInstrumentTypes = ["Ephemeris"];
    
	private $excludeIns = ["GIFWALK", "Unknown"], $excludeGroup = ["THEMIS", "ARTEMIS", "NOAA"];
	
	// not in "https://heliophysicsdata.sci.gsfc.nasa.gov/queries/CDAWeb_SPASE.xql" List;
	// FOR INFO : Excluded automatically
   private $excludeDatasets = ["DMSP_R0_SSJ4", "DMSP_R0_SSIES", "I7_R0_LEPEDEA", "I8_R0_LEPEDEA" ];
   
	private $DDserverXml, $DDserverXmlName, $DDserverDir;
	
	public $dir;
	
	public function html_encode($param) 
	{
		$pairs = array(" " => "%20"); 
		
		return strtr($param,$pairs); 
	}
	
	public function init() 	
	{		   							
		$this->getAllSpaseDatasets();		
	}			 
		
   private function openConnection() 
   {	
		$this->ch = curl_init();			 
		curl_setopt($this->ch, CURLOPT_RETURNTRANSFER, 1);
		curl_setopt($this->ch, CURLOPT_TIMEOUT, 60);		 
	} 
		  
	private function closeConnection()
	{
		curl_close($this->ch);	
	}
	
	public function initProxy() 
	{	 
		$this->dir = getenv("DDBASE")."/../INFO";
		define('RemoteData', $this->dir."/bases/");
		
	//	if (!is_dir(RemoteData)) mkdir(RemoteData, 0755, true);
		if (!is_dir($this->dir."/bases/".$this->baseID))
			mkdir($this->dir."/bases/".$this->baseID, 0755, true);
					
		$this->DDserverDir = $this->dir."/DDServer/".$this->baseID;
		
		if (!is_dir($this->DDserverDir))
			mkdir($this->DDserverDir, 0755, true);		
		
		$this->init();
		
		$this->getRemoteTree();		
	}	 
	
	protected function setDataViewURL()
	{
		curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$restUrl."/dataviews");
		$this->res = new DomDocument();	
		$this->res->loadXML(curl_exec($this->ch));
		$this->dataViewURL = $this->res->getElementsByTagName('EndpointAddress')->item(0)->nodeValue;
	}
		
	protected function getRemoteTree()	
	{		
		$this->openConnection();
		
		$this->setDataViewURL();     
      $this->obsGroupsIds = array();           
           
      foreach ( $this->validInstrumentTypes as $insType)
      {		
			curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/observatoryGroupsAndInstruments?instrumentType=".$insType);			   
			echo $insType.PHP_EOL;
			$this->res->loadXML(curl_exec($this->ch));
 	//	   $this->res->save($insType.".xml");
 		   $observatoryGroups = $this->res->getElementsByTagName("ObservatoryGroupInstrumentDescription");
 		   
 		   foreach ( $observatoryGroups as $obsGroup )
 		   {
				$obsGroupId = $obsGroup->getElementsByTagName('Name')->item(0)->nodeValue;
				
				// Exclude some Missions
				if (in_array ($obsGroupId, $this->excludeGroup )) continue;
				
				echo "   ".$obsGroupId.PHP_EOL;
				$observatories =  $obsGroup->getElementsByTagName("ObservatoryInstruments");
				if ( $observatories->length == 0 )
				{
					echo '      No observatories '.PHP_EOL;
					if ($obsGroup->getElementsByTagName("InstrumentDescription")->length > 0)
							echo '      CHECK INSTRUMENTS '.PHP_EOL;				   
					continue;
				}
				$obsIds = array();
				foreach ( $observatories as $obs )
				{
					$obsId = $obs->getElementsByTagName('Name')->item(0)->nodeValue;
					echo "      ".$obsId.PHP_EOL;
// 				<InstrumentDescription>
// 				  <Name>MAG</Name>
// 				  <ShortDescription>ACE Magnetic Field Instrument</ShortDescription>
// 				  <LongDescription>ACE Magnetic Field Instrument</LongDescription>
// 			   </InstrumentDescription>
					$instruments =  $obs->getElementsByTagName("InstrumentDescription");
					
					$insIds = array();
					
					foreach ( $instruments as $ins )
					{
						$insId = $ins->getElementsByTagName('Name')->item(0)->nodeValue;
						
						// Exclude some 'non-AMDA' staff
						if (in_array ($insId, $this->excludeIns )) continue;
						
						$dsIds = $this->getDatasetByObservatoryAndInstrument($obsGroupId, $obsId, $insId);
					   
					   $insIds[$insId] = $dsIds;
  
					}
					
					$obsIds[$obsId] = $insIds;
				}
				
				$this->obsGroupsIds[$obsGroupId][$insType] = $obsIds;
			}
		}	
		
	//	print_r($this->obsGroupsIds);
	//	$this->closeConnection();		
	}
	
	protected function getDatasetByObservatoryAndInstrument($obsGroupId,  $obsId, $insId)
	{
	//	echo "In getDataset ".$obsId." ".$insId.PHP_EOL;
		
		$grpEncodedId = $this->html_encode($obsGroupId);
		$insEncodedId = $this->html_encode($insId);
		
		$res = new DomDocument("1.0");
		curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets?observatoryGroup=$grpEncodedId&instrument=$insEncodedId");	
		
		$res->loadXML(curl_exec($this->ch));
// 		$this->res->save('dataset.xml');
// 		exit();
		$datasets = $res->getElementsByTagName('DatasetDescription');
		$dsIds = array();
		
		foreach ( $datasets as $ds)
		{
			$obs = $ds->getElementsByTagName('Observatory')->item(0)->nodeValue;
			if ($obs != $obsId ) continue;
			$dsIds[] = $ds->getElementsByTagName('Id')->item(0)->nodeValue;
		}
		
		return $dsIds;  
	}
	
	protected function createMissionNodes()
	{
	   $missionNodes = array();
	   $missionIds = array();
	   // unique values
	   // $this->obsGroupsIds = array_map("unserialize", array_unique(array_map("serialize", $this->obsGroupsIds)));
	   
		foreach ($this->obsGroupsIds as $groupId => $insTypes)
		{
			// echo '  GroupID '.$groupId.PHP_EOL;
			$missionId = $this->baseID.":".$this->param2dd($groupId);
			
			if (!in_array($missionId, $missionIds))
			{
				$missionNode = $this->domAmda->createElement('mission');
				$missionNode->setAttribute("xml:id",$missionId);
				$missionNode->setAttribute("name",$groupId);
				
				$missionIds[] = $missionId;
				$newNode = true;
			}
			else {
				$key = array_search($missionId, $missionIds);
				$missionNode = $missionNodes[$key];	
				$newNode = false;
			}
			
			foreach ($insTypes as $insType => $obss)
			{
				// echo $insType.PHP_EOL;
				
				foreach ( $obss as $obs => $inss)
				{
					// echo $obs.PHP_EOL;
					// OMNI ONLY ONCE
					if (substr($obs,0,4) == "OMNI" && substr($missionId,0,4) != "OMNI")
					{
						//echo '       '.$obs.' '.$missionId.PHP_EOL;
						continue;
					}
					
					$obsId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obs);
					$obsNodes = $missionNode->getElementsByTagName('observatory');
					$newObsNode = true;
					
					foreach ($obsNodes as $node)
					{
						$id = $node->getAttribute("xml:id");
						
						if ($id == $obsId) 
						{
							$obsNode = $node;
							$newObsNode = false;							
							break;
						}
					}
					if ( $newObsNode )
					{
						$obsNode = $this->createObservatoryNode($obs, $groupId);						
					}
					foreach ( $inss as $ins => $dss)
					{	
						$insId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obs).":".$this->param2dd($ins);
						$insNodes = $obsNode->getElementsByTagName('instrument');
						$newInsNode = true;
						
						foreach ($insNodes as $node)
						{
							$id = $node->getAttribute("xml:id");
							
							if ($id == $insId) 
							{
								$insNode = $node;
								$newInsNode = false;
								break;
							}
						}
						if ( $newInsNode )					
							$insNode = $this->createInstrumentNode($ins, $obs, $groupId);							
						 
						foreach ($dss as $ds)
						{	
							$dsId = $this->baseID.":".$ds;
							$dsNodes = $insNode->getElementsByTagName('dataset');
						   $newDsNode = true;
						   
							foreach ($dsNodes as $node)
							{
								$id = $node->getAttribute("xml:id");
								
								if ($id == $dsId) 
								{
									$dsNode = $node;
									$newDsNode = false;
									break;
								}
							}
							
							if ($newDsNode) 
							{
								$this->initDDServerXml($ds,$ins,$obs);
								
								$dsNode =  $this->createDatasetNode($ds);
								if ($dsNode)
								{
									$insNode->appendChild($dsNode);
									$this->saveDDServerXml();
								}
							}
						}
						
						if ($newInsNode && $insNode->hasChildNodes())
							$obsNode->appendChild($insNode);
					}	
					
					if ($newObsNode && $obsNode->hasChildNodes())
					{				  
						$missionNode->appendChild($obsNode);
					}
				}				
				
			}
			
			if ($newNode) 
				$missionNodes[] = $missionNode;
		}
			 
		return $missionNodes;
		
	}
	
	protected function createObservatoryNode($id, $grpId)
	{	
	
		$insNode = $this->domAmda->createElement('observatory');
				
		$insNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($grpId).":".$this->param2dd($id));
		$insNode->setAttribute("name",$id);
	  //	$insNode->setAttribute("description",$ins[1]);
		
		return $insNode;
	}
	
	protected function createInstrumentNode($id, $obsId, $groupId)
	{	
	
		$insNode = $this->domAmda->createElement('instrument');
				
		$insNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsId).":".$id);
		$insNode->setAttribute("name",$id);
	  //	$insNode->setAttribute("description",$ins[1]);
		
		return $insNode;
	}
	
	protected function createDatasetNode($dsId)
	{		
		$dsNode = $this->domAmda->createElement('dataset');
		
		curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json"));
		curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets/?idPattern=".$dsId);			   
			
		$obj = json_decode(curl_exec($this->ch));
      $dataSet = $obj->DatasetDescription;
      
		$dsNode->setAttribute("xml:id",$this->baseID.":".$dsId);
		$dsNode->setAttribute("name", $dsId);
		$startTime = $dataSet[0]->TimeInterval->Start;
		$endTime =  $dataSet[0]->TimeInterval->End;
		$label = $dataSet[0]->Label; 
		$url = $dataSet[0]->DatasetLink[0]->Url;
		$notesUrl = $dataSet[0]->Notes; 
		$piAffilation = $dataSet[0]->PiAffiliation;
		$piName = $dataSet[0]->PiName; 
		$sampling = $this->getDatasetSpaseDescription($dsId);
		$this->updateDDServerXml("GlobalStart",$startTime);
		$this->updateDDServerXml("GlobalStop",$endTime);
	
		// no general description - strange dataset
		if ($sampling == -1)
			return null;
			
		if ($sampling < -1)	
			echo "       !!! $dsId $sampling".PHP_EOL;
		else
			$this->updateDDServerXml("MinSampling",$sampling);	
			
	   $dsNode->setAttribute('spaseUrl',$this->CDAWEB[$dsId]);
	//   $dsNode->setAttribute('masterCdf',$this->existsMasterCdf($dsId));
	   
		$parameterNodes = $this->createParameterNodes($dsId);
		foreach ( $parameterNodes as $parameterNode)
		{
			$dsNode->appendChild($parameterNode);
		}
		
		return $dsNode;
	}
	
	protected function createParameterNodes($dsId)
	{	
		curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json"));
		curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets/".$dsId."/variables");			   
			 
	 	$obj = json_decode(curl_exec($this->ch));
	 	$parameters = $obj->VariableDescription;
	 	 
		$paramNodes = array();
		
		foreach ($parameters as $param)
		{
			$paramNode = $this->domAmda->createElement('parameter');
			$paramNode->setAttribute("xml:id", $this->baseID.":".$dsId.":".$param->Name);
			$paramNode->setAttribute("name", $param->Name);
			
			$paramNodes[] = $paramNode;
		}	
		
		return $paramNodes;
	}
	
	protected function getDatasetSpaseDescription($dsID)
	{
	//	$this->openConnection();		 
		curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/xml"));

		if ( !array_key_exists($dsID, $this->CDAWEB ))
			return -1; // no description in SpaseRegistry
			
		curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$spaseResolver."id=".$this->CDAWEB[$dsID]);
		
		$this->res->loadXML(curl_exec($this->ch));
		//TODO errors
		$messages = $this->res->getElementsByTagName('Message');
		
		if ($messages->length > 0)
		{
			foreach ($messages as $message)
					echo $message->nodeValue.PHP_EOL;
			return -2;   // no description in  www-spase
		}
		
		$instrument = $this->res->getElementsByTagName('InstrumentID');
	   echo "     instrument ".$instrument->item(0)->nodeValue.PHP_EOL;
		
		$cadence = $this->res->getElementsByTagName('Cadence');
		
		if ( $cadence->length == 0)
			return -3; // no cadence in spase xml
			
		$sampling = $this->cadence2sampling($cadence->item(0)->nodeValue);
		
		return $sampling; 	 
	}
	
	public function cadence2sampling($cadence)
	{
		$scale = array('S' => 1, 'M' => 60, 'H' => '3600');
	   
		$value = substr($cadence,2,strlen($cadence)-3);
 
		$units = substr($cadence,-1); 
	   
		$sampling = (int)$value*$scale[$units];
	
		return $sampling;
	}
	
	protected function getAllSpaseDatasets()
	{
		require_once "simple_html_dom.php";
		
		if (file_exists("NumericalData.html")) rename("NumericalData.html","NumericalData.html.bak");
	//	exec("wget -O NumericalData.html ".CDAWebConfigClass::$spaseRegistry);
		//TODO errors
		copy(CDAWebConfigClass::$spaseRegistry, "NumericalData.html");
		
		$html = file_get_html('NumericalData.html');
		$ids = $html->find('td[class="Spase.URL.ProductID"]');

		foreach ($ids as $id)
		{		    
		   $ref = $id->next_sibling()->find('a');
		   $key = $id->find('a');
		   $this->CDAWEB[$key[0]->innertext] = $ref[0]->innertext;		   
		}
		
		echo count($this->CDAWEB).PHP_EOL;		
	}
	
	protected function initDDServerXml($ds, $ins, $obs)
	{
		$this->DDserverXml = new DomDocument("1.0");
		$this->DDserverXmlName =  $this->DDserverDir."/".$ds.".xml";
		$rootNode = $this->DDserverXml->createElement('VI');
		$this->DDserverXml->appendChild($rootNode);
		$rootNode->appendChild($this->DDserverXml->createElement("DataBaseID", $this->baseID));
		$rootNode->appendChild($this->DDserverXml->createElement("Mission", $obs));
		$rootNode->appendChild($this->DDserverXml->createElement("Instrument", $ins));
		$rootNode->appendChild($this->DDserverXml->createElement("DataSetID", $ds));
		$rootNode->appendChild($this->DDserverXml->createElement("GlobalStart"));
		$rootNode->appendChild($this->DDserverXml->createElement("GlobalStop"));
		$rootNode->appendChild($this->DDserverXml->createElement("MinSampling"));	
	}
	
	protected function updateDDServerXml($target, $value)
	{
		$node = $this->DDserverXml->getElementsByTagName($target);
		$node->item(0)->nodeValue = $value;
	}
	
	protected function saveDDServerXml()
	{
		$this->DDserverXml->save($this->DDserverXmlName);
	}
	
	protected function existsMasterCdf($dsId)
	{
		$file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf";
		$file_headers = @get_headers($file);
		
		if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
			$exists = false;
		//	echo "           NO master cdf ".$dsId.PHP_EOL;
		}
		else {
			$exists = true;
		}
		
		return $exists;
	}
		 
	protected function getMasterCdf($dsId)
	{
		$file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf";
		$file_headers = @get_headers($file);
		
		if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
			 return false;
		}		 
		return $file;		 	 
	}
	
	public function getData($ds, $start, $stop)
	{
		$this->openConnection();
		$this->setDataViewURL(); 

	   curl_setopt($this->ch, CURLOPT_URL,$this->dataViewURL."/datasets/$ds/orig_data/$start,$stop/");
	   $res = new DomDocument("1.0");
	   
	   $res->loadXML(curl_exec($this->ch));
	   //TODO errors
	   if ($res->getElementsByTagName("html")->length > 0)
	   {
			echo $res->saveXML();
			exit();
	   }
	   
	   $fileNames = $res->getElementsByTagName("Name");
	   	   
		$nc_prefix = strlen($ds) > RemoteDataCenterClass::$MAX_VI_NAME_LENGTH ? 
				substr(strtolower($ds),0,RemoteDataCenterClass::$MAX_VI_NAME_LENGTH - 1): strtolower($ds);
		$files = array();
	   
		for ($i = 0; $i < $fileNames->length;  $i++)
	   {
			$fileName = $fileNames->item($i);
			$url = $fileName->nodeValue;
			$file_headers = @get_headers($url);
		
			if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
			//TODO  errors
				continue;
			}	
			$temp = explode('/',$url);
			$destination = $temp[count($temp)-1];
			
			if (copy($url, $destination))
			{
				$ncFile = $this->convert2nc($destination);
				if (strlen($ncFile) > RemoteDataCenterClass::$MAX_NAME_LENGTH)
				{
					$ncFileTrue = $nc_prefix."_".time().$i.".nc";
					rename($ncFile, $ncFileTrue);
					$files[] = $ncFileTrue;
				}
				else
				{
					$files[] = $ncFile;
				}
			}
			else
			{
			 //TODO errors
			}
	   }
	   
		$this->closeConnection();
		
		return $files;
	}
	
	protected function convert2nc($file)
	{
		//TODO errors
		system("cdf2nc $file");		 
		$ncFile = str_replace(".cdf", ".nc", $file);
		
 		if (!file_exists($ncFile)) return false; 
		unlink($file);	
		
		return $ncFile;
	}
	
	public function getDatasetInfo($ds)
	{
		$masterCdf = strtolower($ds)."_00000000_v01.cdf";	
		$localCdf = strtolower($ds).".cdf";
		
		if (!copy(CDAWebConfigClass::$masterUrl."/".$masterCdf,  strtolower($ds).".cdf")) return false;
	 
		$infoFile = $this->convert2nc($localCdf);
	  
		return $infoFile;
	}
	protected function setDataCenterAttributes(){}
	protected function makeArgumentsList(){}
} 
?>