<?php
/**
 * @class CDAWEB 
 * @brief  CDAWeb 
 * @details
 */
class CDAWEB extends RemoteDataCenterClass
{
	private $ch, $res, $dataViewUR; 
	private $obsGroupsIds;

	private $spase_res, $insXML;

	
	private $observatoryGroups, $instrumentTypes;
	private $CDAWEB = array(), $masterCDF = array();

	private $header;

	
	// https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/instrumentTypes

	private $validInstrumentTypes = [ "Activity%20Indices", "Electric%20Fields%20(space)","Gamma%20and%20X-Rays",

                                     "Magnetic%20Fields%20(space)", "Particles%20(space)", "Plasma%20and%20Solar%20Wind", 

                                     "Radio%20and%20Plasma%20Waves%20(space)" ];     

//    private $validInstrumentTypes = [ "Radio%20and%20Plasma%20Waves%20(space)" ];                               

//   not used : $validInstrumentTypes = ["Ephemeris"];
    
	private $excludeIns = ["GIFWALK", "Unknown"], $excludeGroup = ["THEMIS", "ARTEMIS", "NOAA"];
	
	// not in "https://heliophysicsdata.sci.gsfc.nasa.gov/queries/CDAWeb_SPASE.xql" List;
	// FOR INFO : Excluded automatically

   private $excludeDatasets = [ "DMSP_R0_SSJ4", "DMSP_R0_SSIES", "I7_R0_LEPEDEA", "I8_R0_LEPEDEA" ];

	protected $DDserverXml, $DDserverXmlName;

	
	public function html_encode($param) 
	{
		$pairs = array(" " => "%20"); 
		
		return strtr($param,$pairs); 
	}
	
	public function init() 	

	{			

		error_log("CDAWEB Proxy creation on ".date("Y-m-d").PHP_EOL,3,log);

		error_log("CDAWEB Proxy creation on ".date("Y-m-d").PHP_EOL,3,err);

		$this->getAllSpaseDatasets();		
	}			 
		
   private function openConnection() 
   {	
		$this->ch = curl_init();			 
		curl_setopt($this->ch, CURLOPT_RETURNTRANSFER, 1);

		curl_setopt($this->ch, CURLOPT_TIMEOUT, 60); 

	} 
		  
	private function closeConnection()
	{
		curl_close($this->ch);	
	}
	

	protected function setDataViewURL()
	{
		curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$restUrl."/dataviews");

		$this->res = new DomDocument();	
		$this->res->loadXML(curl_exec($this->ch));

		
		if ($this->res->getElementsByTagName('EndpointAddress')->length == 0) {
				error_log("Problem connect to ".CDAWebConfigClass::$restUrl."/dataviews", 3, err);
				exit("Problem connect to ".CDAWebConfigClass::$restUrl."/dataviews".PHP_EOL);
		}
		

		$this->dataViewURL = $this->res->getElementsByTagName('EndpointAddress')->item(0)->nodeValue;
	}

	
	// Using CDAS REST APIs get all CDAWEB dataset descriptions

	protected function getRemoteTree()	
	{		
		$this->openConnection();
		
		$this->setDataViewURL();     
      $this->obsGroupsIds = array();           
           

      foreach ($this->validInstrumentTypes as $insType)

      {		
			curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/observatoryGroupsAndInstruments?instrumentType=".$insType);			   
			echo $insType.PHP_EOL;
			$this->res->loadXML(curl_exec($this->ch));

			$observatoryGroups = $this->res->getElementsByTagName("ObservatoryGroupInstrumentDescription");

			foreach ($observatoryGroups as $obsGroup)
			{

				$obsGroupId = $obsGroup->getElementsByTagName('Name')->item(0)->nodeValue;
				
				// Exclude some Missions
				if (in_array ($obsGroupId, $this->excludeGroup )) continue;
				
				echo "   ".$obsGroupId.PHP_EOL;
				$observatories =  $obsGroup->getElementsByTagName("ObservatoryInstruments");
				if ( $observatories->length == 0 )
				{
					echo '      No observatories '.PHP_EOL;

					if ($obsGroup->getElementsByTagName("InstrumentDescription")->length > 0) {
							echo '      CHECK INSTRUMENTS '.PHP_EOL;
							error_log(" CHECK INSTRUMENTS ".$obsGroup.PHP_EOL, 3, err);
					}	

					continue;
				}
				$obsIds = array();
				foreach ( $observatories as $obs )
				{
					$obsId = $obs->getElementsByTagName('Name')->item(0)->nodeValue;
					echo "      ".$obsId.PHP_EOL;
// 				<InstrumentDescription>
// 				  <Name>MAG</Name>
// 				  <ShortDescription>ACE Magnetic Field Instrument</ShortDescription>
// 				  <LongDescription>ACE Magnetic Field Instrument</LongDescription>
// 			   </InstrumentDescription>
					$instruments =  $obs->getElementsByTagName("InstrumentDescription");
					
					$insIds = array();
					
					foreach ( $instruments as $ins )
					{
						$insId = $ins->getElementsByTagName('Name')->item(0)->nodeValue;
						
						// Exclude some 'non-AMDA' staff
						if (in_array ($insId, $this->excludeIns )) continue;
						
						$dsIds = $this->getDatasetByObservatoryAndInstrument($obsGroupId, $obsId, $insId);
					   
					   $insIds[$insId] = $dsIds;
  
					}
					
					$obsIds[$obsId] = $insIds;
				}
				

				$this->obsGroupsIds[$insType][$obsGroupId] = $obsIds;

			}

		}		

	}
	
	protected function getDatasetByObservatoryAndInstrument($obsGroupId,  $obsId, $insId)

	{		

		$grpEncodedId = $this->html_encode($obsGroupId);
		$insEncodedId = $this->html_encode($insId);

		$obsEncodedId = $this->html_encode($obsId);

		
		$res = new DomDocument("1.0");

	//	curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets?observatoryGroup=$grpEncodedId&instrument=$insEncodedId");
	
		curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets?observatory=$obsEncodedId&instrument=$insEncodedId");

		
		$res->loadXML(curl_exec($this->ch));

		$datasets = $res->getElementsByTagName('DatasetDescription');
		$dsIds = array();
		
		foreach ( $datasets as $ds)
		{

		//	$obs = $ds->getElementsByTagName('Observatory')->item(0)->nodeValue;

		//	if ($obs != $obsId ) continue;

			$dsIds[] = $ds->getElementsByTagName('Id')->item(0)->nodeValue;
		}
		
		return $dsIds;  
	}
	

	protected function createObservatoryNode($id, $grpId)
	{	
	

		$obsNode = $this->domAmda->createElement('observatory');

		$obsNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($grpId).":".$this->param2dd($id));
		$obsNode->setAttribute("name",$id);

		return $obsNode;

	}
	
	protected function createInstrumentNode($id, $obsId, $groupId)
	{	
	
		$insNode = $this->domAmda->createElement('instrument');
				
		$insNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsId).":".$id);
		$insNode->setAttribute("name",$id);
	  //	$insNode->setAttribute("description",$ins[1]);
		
		return $insNode;
	}
	
	protected function createDatasetNode($dsId)
	{		
		$dsNode = $this->domAmda->createElement('dataset');
		
		curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json"));
		curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets/?idPattern=".$dsId);			   
			
		$obj = json_decode(curl_exec($this->ch));
      $dataSet = $obj->DatasetDescription;
      
		$dsNode->setAttribute("xml:id",$this->baseID.":".$dsId);
		$dsNode->setAttribute("name", $dsId);
		$startTime = $dataSet[0]->TimeInterval->Start;
		$endTime =  $dataSet[0]->TimeInterval->End;
		$label = $dataSet[0]->Label; 
		$url = $dataSet[0]->DatasetLink[0]->Url;
		$notesUrl = $dataSet[0]->Notes; 
		$piAffilation = $dataSet[0]->PiAffiliation;
		$piName = $dataSet[0]->PiName; 
		$sampling = $this->getDatasetSpaseDescription($dsId);
		$this->updateDDServerXml("GlobalStart",$startTime);
		$this->updateDDServerXml("GlobalStop",$endTime);

		// no general description - strange dataset

		if ($sampling == -1) {
			error_log("! No general description : ".$dsId.PHP_EOL,3,err);

			return null;

		}

		if ($sampling < -1)	{

			echo "       !!! $dsId $sampling".PHP_EOL;

			error_log("! No sampling : ".$dsId." : ".$sampling.PHP_EOL,3,err);
		}

		else
			$this->updateDDServerXml("MinSampling",$sampling);	

		$dsNode->setAttribute('spaseUrl',$this->CDAWEB[$dsId]);

	//   $dsNode->setAttribute('masterCdf',$this->existsMasterCdf($dsId));

		$dsNode->setAttribute("desc", "$label; $startTime - $endTime");
		$dsNode->setAttribute("start", $startTime); 
		$dsNode->setAttribute("stop", $stopTime); 

	   
		$parameterNodes = $this->createParameterNodes($dsId);
		foreach ( $parameterNodes as $parameterNode)
		{
			$dsNode->appendChild($parameterNode);
		}
		
		return $dsNode;
	}
	
	protected function createParameterNodes($dsId)
	{	
		curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json"));
		curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets/".$dsId."/variables");			   
			 
	 	$obj = json_decode(curl_exec($this->ch));
	 	$parameters = $obj->VariableDescription;

		$paramNodes = array();
		
		foreach ($parameters as $param)
		{
			$paramNode = $this->domAmda->createElement('parameter');
			$paramNode->setAttribute("xml:id", $this->baseID.":".$dsId.":".$param->Name);
			$paramNode->setAttribute("name", $param->Name);
			
			$paramNodes[] = $paramNode;
		}	
		
		return $paramNodes;
	}
	
	protected function getDatasetSpaseDescription($dsID)
	{
	//	$this->openConnection();		 
		curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/xml"));

		if ( !array_key_exists($dsID, $this->CDAWEB ))
			return -1; // no description in SpaseRegistry
			
		curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$spaseResolver."id=".$this->CDAWEB[$dsID]);

		if (!$this->spase_res->loadXML(curl_exec($this->ch)))
			return -100;
 
		$messages = $this->spase_res->getElementsByTagName('Message');

		
		if ($messages->length > 0)
		{
			foreach ($messages as $message)
					echo $message->nodeValue.PHP_EOL;
			return -2;   // no description in  www-spase
		}
		

// 		$instrument = $this->spase_res->getElementsByTagName('InstrumentID');
// 	   echo "     instrument ".$instrument->item(0)->nodeValue.PHP_EOL;

		$cadence = $this->spase_res->getElementsByTagName('Cadence');

		if ($cadence->length == 0)

			return -3; // no cadence in spase xml
			
		$sampling = $this->cadence2sampling($cadence->item(0)->nodeValue);
		
		return $sampling; 	 
	}
	
	public function cadence2sampling($cadence)
	{
		$scale = array('S' => 1, 'M' => 60, 'H' => '3600');
	   
		$value = substr($cadence,2,strlen($cadence)-3);
 
		$units = substr($cadence,-1); 
	   

		$sampling = floatval($value)*$scale[$units];
		

		return $sampling;
	}
	

	// Get IDs and SPASE URLs of all CDAWEB SPASE-defined datasets 

	protected function getAllSpaseDatasets()
	{
		require_once "simple_html_dom.php";
		

		if (file_exists($this->location."/NumericalData.html")) 
				rename($this->location."/NumericalData.html",$this->location."/NumericalData.html.bak");

		// if cannot reach CDAWEB Spase Registry use an old file

		if (!copy(CDAWebConfigClass::$spaseRegistry, $this->location."/NumericalData.html"))
				copy($this->location."/NumericalData.html.bak", $this->location."/NumericalData.html");

		$html = file_get_html($this->location.'/NumericalData.html');

		$ids = $html->find('td[class="Spase.URL.ProductID"]');

		foreach ($ids as $id)
		{		    
		   $ref = $id->next_sibling()->find('a');
		   $key = $id->find('a');
		   $this->CDAWEB[$key[0]->innertext] = $ref[0]->innertext;		   
		}

		error_log("SPASE-defined CDAWEB datasets from ".CDAWebConfigClass::$spaseRegistry." : ".count($this->CDAWEB).PHP_EOL,3,log);

		echo "SPASE-defined CDAWEB datasets : ".count($this->CDAWEB).PHP_EOL;

	}
	
	protected function initDDServerXml($ds, $ins, $obs)
	{
		$this->DDserverXml = new DomDocument("1.0");
		$this->DDserverXmlName =  $this->DDserverDir."/".$ds.".xml";
		$rootNode = $this->DDserverXml->createElement('VI');
		$this->DDserverXml->appendChild($rootNode);
		$rootNode->appendChild($this->DDserverXml->createElement("DataBaseID", $this->baseID));
		$rootNode->appendChild($this->DDserverXml->createElement("Mission", $obs));
		$rootNode->appendChild($this->DDserverXml->createElement("Instrument", $ins));
		$rootNode->appendChild($this->DDserverXml->createElement("DataSetID", $ds));
		$rootNode->appendChild($this->DDserverXml->createElement("GlobalStart"));
		$rootNode->appendChild($this->DDserverXml->createElement("GlobalStop"));

		$rootNode->appendChild($this->DDserverXml->createElement("MinSampling"));
		// fill value by default - if different - add into param description

		$rootNode->appendChild($this->DDserverXml->createElement("FillValue", -1.0E31));

	}
	
	protected function updateDDServerXml($target, $value)
	{
		$node = $this->DDserverXml->getElementsByTagName($target);
		$node->item(0)->nodeValue = $value;
	}
	
	protected function saveDDServerXml()
	{
		$this->DDserverXml->save($this->DDserverXmlName);
	}
	
	protected function existsMasterCdf($dsId)
	{
		$file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf";
		$file_headers = @get_headers($file);
		
		if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
			$exists = false;

			error_log("INFO:   no master CDF ".$dsId.PHP_EOL,3,log);

		}
		else {
			$exists = true;
		}
		
		return $exists;
	}
		 
	protected function getMasterCdf($dsId)
	{
		$file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf";
		$file_headers = @get_headers($file);
		
		if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
			 return false;
		}		 
		return $file;		 	 
	}
	
	public function getData($ds, $start, $stop)
	{
		$this->openConnection();
		$this->setDataViewURL(); 

	   curl_setopt($this->ch, CURLOPT_URL,$this->dataViewURL."/datasets/$ds/orig_data/$start,$stop/");
	   $res = new DomDocument("1.0");
	   
	   $res->loadXML(curl_exec($this->ch));

	   if ($res->getElementsByTagName("html")->length > 0)
	   {

			error_log("ERROR no response for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
			error_log($res->saveXML(),3,err);

			exit();
	   }
	   
	   $fileNames = $res->getElementsByTagName("Name");
	   	   
		$nc_prefix = strlen($ds) > RemoteDataCenterClass::$MAX_VI_NAME_LENGTH ? 
				substr(strtolower($ds),0,RemoteDataCenterClass::$MAX_VI_NAME_LENGTH - 1): strtolower($ds);
		$files = array();
	   
		for ($i = 0; $i < $fileNames->length;  $i++)
	   {
			$fileName = $fileNames->item($i);
			$url = $fileName->nodeValue;
			$file_headers = @get_headers($url);
		
			if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {

				error_log("ERROR  404 Not Found for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);

				continue;
			}	
			$temp = explode('/',$url);
			$destination = $temp[count($temp)-1];
			
			if (copy($url, $destination))
			{
				$ncFile = $this->convert2nc($destination);
				if (strlen($ncFile) > RemoteDataCenterClass::$MAX_NAME_LENGTH)
				{
					$ncFileTrue = $nc_prefix."_".time().$i.".nc";
					rename($ncFile, $ncFileTrue);
					$files[] = $ncFileTrue;
				}
				else
				{
					$files[] = $ncFile;
				}
			}
			else
			{

				error_log("ERROR cannot copy files : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);

			}
	   }
	   
		$this->closeConnection();
		
		return $files;
	}
	
	protected function convert2nc($file)
	{

		system("cdfnew2nc $file");		 

		$ncFile = str_replace(".cdf", ".nc", $file);
		

 		if (!file_exists($ncFile)) {
			error_log("ERROR while converting ".$file.PHP_EOL,3,err);
			return false; 
 		}

		unlink($file);	
		
		return $ncFile;
	}
	

	public function getDatasetInfo($ds)
	{
		$masterCdf = strtolower($ds)."_00000000_v01.cdf";	
		$localCdf = strtolower($ds).".cdf";
		
		if (!copy(CDAWebConfigClass::$masterUrl."/".$masterCdf,  strtolower($ds).".cdf")) return false;
	 
		$infoFile = $this->convert2nc($localCdf);
	  
		return $infoFile;
	}

	
	public function makeProxy()
	{	 
		$this->spase_res = new DomDocument("1.0");
		$this->domAmda = new DOMDocument('1.0', 'utf-8');
		$this->domAmda->formatOutput = TRUE;
		$this->domAmda->preserveWhiteSpace = FALSE;
		
		$dataRoot = $this->domAmda->createElement('dataRoot');
		$dataRoot->setAttribute('xml:id', 'myRemoteData-treeRootNode');
		$this->domAmda->appendChild($dataRoot);	
		
		$this->dataCenter=$this->domAmda->createElement('dataCenter');

	 	$this->setDataCenterAttributes();
	 	

		$dataRoot->appendChild($this->dataCenter);

		$first = true;
		

		foreach ($this->obsGroupsIds as $insType => $groupIds)
		{		

			// getElementByID doesn't work on newly created Document
			if (!$first) {
				$this->domAmda->loadXML($this->domAmda->saveXML());			
				$this->dataCenter = $this->domAmda->getElementById($this->baseID);
			}

			foreach ($groupIds as $groupId => $obss)
			{			

				$obsNodes = array();

				foreach ($obss as $obs => $inss)

				{	
					// ! observatory OMNI is taken for observatoriesGroup OMNI only !

					if (substr($obs,0,4) == "OMNI" && substr($groupId,0,4) != "OMNI"){
						echo 'OMNI      '.$obs.' '.$missionId.PHP_EOL;
						continue;

					} 

					$insNodes = array();
					// Create instrument nodes

					foreach ($inss as $ins => $dss)
					{

						$dsNodes = array();
						// Create datasets nodes

						foreach ($dss as $ds)
						{	

							$dsId = $this->baseID.":".$ds;

							if (!$this->domAmda->getElementById($dsId)){
								$this->initDDServerXml($ds,$ins,$obs);
								$dsNode = $this->createDatasetNode($ds);
								if ($dsNode){
									$this->saveDDServerXml();									
									$dsNodes[] = $dsNode;
								}
							}	
						} // foreach ($dss as $ds)

						if ($dsNodes)
						{

							// last $spase_res : instrument should be the same
							$insSpaseId = $this->getInstrumentSpase();
							if (!$insSpaseId ) 
								$insSpaseId = $ins;
								

							$obsSpaseId = strtolower($this->getObservatorySpase());

							if (!$obsSpaseId ) 

								$obsSpaseId = strtolower($obs);

							
							$insId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId).":".$this->param2dd($insSpaseId);

							
							if (!($insNode = $this->domAmda->getElementById($insId))){

								$insNode = $this->createInstrumentNode($insSpaseId, $obsSpaseId, $groupId);

							} 

							foreach ($dsNodes as $dsNode){
								$insNode->appendChild($dsNode);
							}

							$insNodes[] = $insNode;

						}
					} // foreach ($inss as $ins => $dss)

					if ($insNodes)
					{					

						$obsId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId);

						if (!($obsNode = $this->domAmda->getElementById($obsId))){   

							$obsNode = $this->createObservatoryNode($obsSpaseId, $groupId);

						}
						
						foreach ($insNodes as $insNode){
							$obsNode->appendChild($insNode);
						}

						
						$obsNodes[] = $obsNode;

					}
				} // foreach ($obss as $obs => $inss)

				if ($obsNodes)
				{

					$missionId = $this->baseID.":".$this->param2dd($groupId);
					
					if (!($missionNode = $this->domAmda->getElementById($missionId))){
						$missionNode = $this->createMissionNode($groupId);
						$this->dataCenter->appendChild($missionNode);

					}

					foreach ($obsNodes as $obsNode)
					{

						// observatory == mission

						if (strtolower($obsNode->getAttribute('name')) == strtolower($missionNode->getAttribute('name'))) {

							$insNodes = $obsNode->getElementsByTagName("instrument");
							foreach ($insNodes as $insNode)
								$missionNode->appendChild($insNode);
						}
						else 
							$missionNode->appendChild($obsNode);
					}	

				}

			}
			$first = false;

		}

		
		$this->closeConnection();

	}
	

	protected function createMissionNode($groupId)
	{

		$missionId = $this->baseID.":".$this->param2dd($groupId);
		$missionNode = $this->domAmda->createElement('mission');
		$missionNode->setAttribute("xml:id",$missionId);
		$missionNode->setAttribute("name",$groupId);
		
		return $missionNode;
	}
	

	protected function getInstrumentSpase()
	{	

		$this->insXML = new DomDocument("1.0");

		
		if ($this->spase_res){
			$instrument = $this->spase_res->getElementsByTagName('InstrumentID');
			
			if ($instrument->length > 0) {

				
				curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$spaseResolver."id=".$instrument->item(0)->nodeValue);
				
				$this->insXML->loadXML(curl_exec($this->ch));
				

				return $this->getIdFromSpase($instrument->item(0)->nodeValue);

				//InstrumentType
				//spase://SMWG/Observatory/ACE
			} 
		}	
		
		return null;
	}
	

	protected function getObservatorySpase()
	{

		if ($this->insXML){
			$observatory = $this->insXML->getElementsByTagName('ObservatoryID');
			
			if ($observatory->length > 0) {
				return $this->getIdFromSpase($observatory->item(0)->nodeValue); 			
			} 
		}	
		
		return null;
	}
	

	protected function setDataCenterAttributes(){	
		$this->dataCenter->setAttribute('xml:id', $this->baseID);
	//	$this->dataCenter->setIdAttribute('xml:id', true);
		$this->dataCenter->setAttribute('name', $this->baseID);		
	}
	

	protected function makeArgumentsList(){}

} 
?>