Blame view

src/REMOTEDATA/CDAWEB.php 22.1 KB
41792000   Elena.Budnik   new REMOTEDATA
1
2
3
4
5
6
7
8
9
10
<?php
/**
 * @class CDAWEB 
 * @brief  CDAWeb 
 * @details
 */
class CDAWEB extends RemoteDataCenterClass
{
	private $ch, $res, $dataViewUR; 
	private $obsGroupsIds;
7216c535   Elena.Budnik   added samplingCDA...
11
	private $spase_res, $insXML, $xp = null;
41792000   Elena.Budnik   new REMOTEDATA
12
13
14
	
	private $observatoryGroups, $instrumentTypes;
	private $CDAWEB = array(), $masterCDF = array();
05fbcab0   Elena.Budnik   DDServices reorga...
15
	private $header;
41792000   Elena.Budnik   new REMOTEDATA
16
17
	
	// https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/instrumentTypes
1b0e0631   Elena.Budnik   unique name for obs
18
	private $validInstrumentTypes = [ "Activity%20Indices", "Electric%20Fields%20(space)","Gamma%20and%20X-Rays",
41792000   Elena.Budnik   new REMOTEDATA
19
                                     "Magnetic%20Fields%20(space)", "Particles%20(space)", "Plasma%20and%20Solar%20Wind", 
1b0e0631   Elena.Budnik   unique name for obs
20
                                     "Radio%20and%20Plasma%20Waves%20(space)" ];     
775592ff   Elena.Budnik   bug in insNodes a...
21
//    private $validInstrumentTypes = [ "Radio%20and%20Plasma%20Waves%20(space)" ];                               
155b6b3c   Elena.Budnik   other approach to...
22
                                     
41792000   Elena.Budnik   new REMOTEDATA
23
24
//   not used : $validInstrumentTypes = ["Ephemeris"];
    
7216c535   Elena.Budnik   added samplingCDA...
25
	private $excludeIns = ["GIFWALK", "Unknown"], $excludeGroup = ["THEMIS", "ARTEMIS", "NOAA", "GPS", "TSS-1R", "IBEX", "SOHO"];
41792000   Elena.Budnik   new REMOTEDATA
26
27
28
	
	// not in "https://heliophysicsdata.sci.gsfc.nasa.gov/queries/CDAWeb_SPASE.xql" List;
	// FOR INFO : Excluded automatically
7216c535   Elena.Budnik   added samplingCDA...
29
    private $excludeDatasets = [ "DMSP_R0_SSJ4", "DMSP_R0_SSIES", "I7_R0_LEPEDEA", "I8_R0_LEPEDEA" ];
41792000   Elena.Budnik   new REMOTEDATA
30
   
d4071e27   Elena.Budnik   makeProxy with en...
31
	protected $DDserverXml, $DDserverXmlName;
41792000   Elena.Budnik   new REMOTEDATA
32
	
41792000   Elena.Budnik   new REMOTEDATA
33
34
35
36
37
38
39
40
41
	
	public function html_encode($param) 
	{
		$pairs = array(" " => "%20"); 
		
		return strtr($param,$pairs); 
	}
	
	public function init() 	
8c0d5a9f   Elena.Budnik   correct location ...
42
	{			
35322ea6   Elena.Budnik   case of absence o...
43
44
		error_log("CDAWEB Proxy creation on ".date("Y-m-d\TH:i:s").PHP_EOL,3,log);
		error_log("CDAWEB Proxy creation on ".date("Y-m-d\TH:i:s").PHP_EOL,3,err);
d1327dc0   Benjamin Renard   Fix RemoteData ac...
45
46

		$this->initStreamContext();
05fbcab0   Elena.Budnik   DDServices reorga...
47
		
7216c535   Elena.Budnik   added samplingCDA...
48
49
50
51
52
53
54
55
56
57
		$this->getAllSpaseDatasets();
		
		// predefined samplings
		if (file_exists(RemoteData."/samplingCDAWEB.xml")) {
            $samplingsDom = new DomDocument("1.0");
            if ($samplingsDom->load(RemoteData."/samplingCDAWEB.xml")) {
                $this->xp = new domxpath($samplingsDom);
            }
		}
	} 
41792000   Elena.Budnik   new REMOTEDATA
58
		
d1327dc0   Benjamin Renard   Fix RemoteData ac...
59
60
	private function openConnection() 
	{	
41792000   Elena.Budnik   new REMOTEDATA
61
62
		$this->ch = curl_init();			 
		curl_setopt($this->ch, CURLOPT_RETURNTRANSFER, 1);
d1327dc0   Benjamin Renard   Fix RemoteData ac...
63
64
65
66
67
68
69
70
71
72
		curl_setopt($this->ch, CURLOPT_TIMEOUT, 60);
		// Add proxy definition
		$PROXY_HOST=getenv('PROXY_HOST');
		$PROXY_USERPWD=getenv('PROXY_USERPWD');
		if (!empty($PROXY_HOST)) {
			curl_setopt($this->ch, CURLOPT_PROXY, getenv('PROXY_HOST'));
			if (!empty($PROXY_USERPWD)) {
				curl_setopt($this->ch, CURLOPT_PROXYUSERPWD, getenv('PROXY_USERPWD'));
			}
		}
41792000   Elena.Budnik   new REMOTEDATA
73
74
75
76
77
78
	} 
		  
	private function closeConnection()
	{
		curl_close($this->ch);	
	}
d1327dc0   Benjamin Renard   Fix RemoteData ac...
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96

	private function initStreamContext()
	{
		$PROXY_HOST=getenv('PROXY_HOST');
		$PROXY_USERPWD=getenv('PROXY_USERPWD');
		if (!empty($PROXY_HOST)) {
			$context = array(
				'http' => array(
					'proxy' => "tcp://$PROXY_HOST",
					'request_fulluri' => true,
				),
			);
			if (!empty($PROXY_USERPWD)) {
				$context['http']['header'] = "Proxy-Authorization: Basic ".base64_encode($PROXY_USERPWD);
                        }
			stream_context_set_default($context);
		}
	}
41792000   Elena.Budnik   new REMOTEDATA
97
	
41792000   Elena.Budnik   new REMOTEDATA
98
99
100
	protected function setDataViewURL()
	{
		curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$restUrl."/dataviews");
d4071e27   Elena.Budnik   makeProxy with en...
101
		
41792000   Elena.Budnik   new REMOTEDATA
102
103
		$this->res = new DomDocument();	
		$this->res->loadXML(curl_exec($this->ch));
775592ff   Elena.Budnik   bug in insNodes a...
104
105
106
107
108
109
		
		if ($this->res->getElementsByTagName('EndpointAddress')->length == 0) {
				error_log("Problem connect to ".CDAWebConfigClass::$restUrl."/dataviews", 3, err);
				exit("Problem connect to ".CDAWebConfigClass::$restUrl."/dataviews".PHP_EOL);
		}
		
41792000   Elena.Budnik   new REMOTEDATA
110
111
		$this->dataViewURL = $this->res->getElementsByTagName('EndpointAddress')->item(0)->nodeValue;
	}
d4071e27   Elena.Budnik   makeProxy with en...
112
113
	
	// Using CDAS REST APIs get all CDAWEB dataset descriptions
41792000   Elena.Budnik   new REMOTEDATA
114
115
116
117
118
119
120
	protected function getRemoteTree()	
	{		
		$this->openConnection();
		
		$this->setDataViewURL();     
      $this->obsGroupsIds = array();           
           
775592ff   Elena.Budnik   bug in insNodes a...
121
      foreach ($this->validInstrumentTypes as $insType)
41792000   Elena.Budnik   new REMOTEDATA
122
123
124
125
      {		
			curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/observatoryGroupsAndInstruments?instrumentType=".$insType);			   
			echo $insType.PHP_EOL;
			$this->res->loadXML(curl_exec($this->ch));
775592ff   Elena.Budnik   bug in insNodes a...
126
127

			$observatoryGroups = $this->res->getElementsByTagName("ObservatoryGroupInstrumentDescription");
41792000   Elena.Budnik   new REMOTEDATA
128
 		   
775592ff   Elena.Budnik   bug in insNodes a...
129
130
			foreach ($observatoryGroups as $obsGroup)
			{
41792000   Elena.Budnik   new REMOTEDATA
131
132
133
134
135
136
137
138
139
140
				$obsGroupId = $obsGroup->getElementsByTagName('Name')->item(0)->nodeValue;
				
				// Exclude some Missions
				if (in_array ($obsGroupId, $this->excludeGroup )) continue;
				
				echo "   ".$obsGroupId.PHP_EOL;
				$observatories =  $obsGroup->getElementsByTagName("ObservatoryInstruments");
				if ( $observatories->length == 0 )
				{
					echo '      No observatories '.PHP_EOL;
8c0d5a9f   Elena.Budnik   correct location ...
141
142
143
144
					if ($obsGroup->getElementsByTagName("InstrumentDescription")->length > 0) {
							echo '      CHECK INSTRUMENTS '.PHP_EOL;
							error_log(" CHECK INSTRUMENTS ".$obsGroup.PHP_EOL, 3, err);
					}	
41792000   Elena.Budnik   new REMOTEDATA
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
					continue;
				}
				$obsIds = array();
				foreach ( $observatories as $obs )
				{
					$obsId = $obs->getElementsByTagName('Name')->item(0)->nodeValue;
					echo "      ".$obsId.PHP_EOL;
// 				<InstrumentDescription>
// 				  <Name>MAG</Name>
// 				  <ShortDescription>ACE Magnetic Field Instrument</ShortDescription>
// 				  <LongDescription>ACE Magnetic Field Instrument</LongDescription>
// 			   </InstrumentDescription>
					$instruments =  $obs->getElementsByTagName("InstrumentDescription");
					
					$insIds = array();
					
					foreach ( $instruments as $ins )
					{
						$insId = $ins->getElementsByTagName('Name')->item(0)->nodeValue;
						
						// Exclude some 'non-AMDA' staff
						if (in_array ($insId, $this->excludeIns )) continue;
						
						$dsIds = $this->getDatasetByObservatoryAndInstrument($obsGroupId, $obsId, $insId);
					   
					   $insIds[$insId] = $dsIds;
  
					}
					
					$obsIds[$obsId] = $insIds;
				}
				
1b0e0631   Elena.Budnik   unique name for obs
177
				$this->obsGroupsIds[$insType][$obsGroupId] = $obsIds;
41792000   Elena.Budnik   new REMOTEDATA
178
			}
05fbcab0   Elena.Budnik   DDServices reorga...
179
		}		
41792000   Elena.Budnik   new REMOTEDATA
180
181
182
	}
	
	protected function getDatasetByObservatoryAndInstrument($obsGroupId,  $obsId, $insId)
8c0d5a9f   Elena.Budnik   correct location ...
183
	{		
41792000   Elena.Budnik   new REMOTEDATA
184
185
		$grpEncodedId = $this->html_encode($obsGroupId);
		$insEncodedId = $this->html_encode($insId);
05fbcab0   Elena.Budnik   DDServices reorga...
186
		$obsEncodedId = $this->html_encode($obsId);
41792000   Elena.Budnik   new REMOTEDATA
187
188
		
		$res = new DomDocument("1.0");
05fbcab0   Elena.Budnik   DDServices reorga...
189
190
191
	//	curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets?observatoryGroup=$grpEncodedId&instrument=$insEncodedId");
	
		curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets?observatory=$obsEncodedId&instrument=$insEncodedId");
41792000   Elena.Budnik   new REMOTEDATA
192
193
		
		$res->loadXML(curl_exec($this->ch));
05fbcab0   Elena.Budnik   DDServices reorga...
194

41792000   Elena.Budnik   new REMOTEDATA
195
196
197
198
199
		$datasets = $res->getElementsByTagName('DatasetDescription');
		$dsIds = array();
		
		foreach ( $datasets as $ds)
		{
05fbcab0   Elena.Budnik   DDServices reorga...
200
		//	$obs = $ds->getElementsByTagName('Observatory')->item(0)->nodeValue;
8c0d5a9f   Elena.Budnik   correct location ...
201
		//	if ($obs != $obsId ) continue;
41792000   Elena.Budnik   new REMOTEDATA
202
203
204
205
206
207
			$dsIds[] = $ds->getElementsByTagName('Id')->item(0)->nodeValue;
		}
		
		return $dsIds;  
	}
	
41792000   Elena.Budnik   new REMOTEDATA
208
209
210
	protected function createObservatoryNode($id, $grpId)
	{	
	
155b6b3c   Elena.Budnik   other approach to...
211
		$obsNode = $this->domAmda->createElement('observatory');
41792000   Elena.Budnik   new REMOTEDATA
212
				
155b6b3c   Elena.Budnik   other approach to...
213
214
		$obsNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($grpId).":".$this->param2dd($id));
		$obsNode->setAttribute("name",$id);
41792000   Elena.Budnik   new REMOTEDATA
215
		
155b6b3c   Elena.Budnik   other approach to...
216
		return $obsNode;
41792000   Elena.Budnik   new REMOTEDATA
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
	}
	
	protected function createInstrumentNode($id, $obsId, $groupId)
	{	
	
		$insNode = $this->domAmda->createElement('instrument');
				
		$insNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsId).":".$id);
		$insNode->setAttribute("name",$id);
	  //	$insNode->setAttribute("description",$ins[1]);
		
		return $insNode;
	}
	
	protected function createDatasetNode($dsId)
	{		
		$dsNode = $this->domAmda->createElement('dataset');
		
		curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json"));
873ce7b2   Elena.Budnik   CDAWEB exclude so...
236
		curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets?idPattern=".$dsId);			   
41792000   Elena.Budnik   new REMOTEDATA
237
238
			
		$obj = json_decode(curl_exec($this->ch));
873ce7b2   Elena.Budnik   CDAWEB exclude so...
239
        $dataSet = $obj->DatasetDescription;
41792000   Elena.Budnik   new REMOTEDATA
240
241
242
243
244
245
246
247
248
249
250
251
252
      
		$dsNode->setAttribute("xml:id",$this->baseID.":".$dsId);
		$dsNode->setAttribute("name", $dsId);
		$startTime = $dataSet[0]->TimeInterval->Start;
		$endTime =  $dataSet[0]->TimeInterval->End;
		$label = $dataSet[0]->Label; 
		$url = $dataSet[0]->DatasetLink[0]->Url;
		$notesUrl = $dataSet[0]->Notes; 
		$piAffilation = $dataSet[0]->PiAffiliation;
		$piName = $dataSet[0]->PiName; 
		$sampling = $this->getDatasetSpaseDescription($dsId);
		$this->updateDDServerXml("GlobalStart",$startTime);
		$this->updateDDServerXml("GlobalStop",$endTime);
1b0e0631   Elena.Budnik   unique name for obs
253
		
41792000   Elena.Budnik   new REMOTEDATA
254
		// no general description - strange dataset
8c0d5a9f   Elena.Budnik   correct location ...
255
256
		if ($sampling == -1) {
			error_log("! No general description : ".$dsId.PHP_EOL,3,err);
41792000   Elena.Budnik   new REMOTEDATA
257
			return null;
8c0d5a9f   Elena.Budnik   correct location ...
258
		}
41792000   Elena.Budnik   new REMOTEDATA
259
			
7216c535   Elena.Budnik   added samplingCDA...
260
261
		if ($sampling <= 0)	{
            // check if there is file with predefined samplings
873ce7b2   Elena.Budnik   CDAWEB exclude so...
262
            echo "       !!! $dsId sampling = $sampling".PHP_EOL;
7216c535   Elena.Budnik   added samplingCDA...
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
            if ($this->xp) {
                $set = $this->xp->query("//dataset[@xml:id='".$dsId."']");
                if ($set->length > 0) {
                    $sampling = $set->item(0)->nodeValue;
                    echo "from samplingCDAWEB : ".$dsId." : ".$sampling.PHP_EOL;
                    $this->updateDDServerXml("MinSampling",$sampling);	
                }  else {                    
                    echo " No dataset in samplingCDAWEB defined".PHP_EOL;
                    error_log("! No sampling and No dataset in samplingCDAWEB defined : ".$dsId." : ".$sampling.PHP_EOL,3,err);
                }
            } else {
                echo " No xpath ".PHP_EOL;
                error_log("! No sampling and No xpath defined : ".$dsId." : ".$sampling.PHP_EOL,3,err);
            }
			
		}
873ce7b2   Elena.Budnik   CDAWEB exclude so...
279
        else 
41792000   Elena.Budnik   new REMOTEDATA
280
			$this->updateDDServerXml("MinSampling",$sampling);	
1b0e0631   Elena.Budnik   unique name for obs
281
		
155b6b3c   Elena.Budnik   other approach to...
282
		$dsNode->setAttribute('spaseUrl',$this->CDAWEB[$dsId]);
41792000   Elena.Budnik   new REMOTEDATA
283
	//   $dsNode->setAttribute('masterCdf',$this->existsMasterCdf($dsId));
155b6b3c   Elena.Budnik   other approach to...
284
		$dsNode->setAttribute("desc", "$label; $startTime - $endTime");
21b939db   Elena.Budnik   dataStart, dataSt...
285
286
		$dsNode->setAttribute("dataStart", $startTime); 
		$dsNode->setAttribute("dataStop", $endTime); 
41792000   Elena.Budnik   new REMOTEDATA
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
	   
		$parameterNodes = $this->createParameterNodes($dsId);
		foreach ( $parameterNodes as $parameterNode)
		{
			$dsNode->appendChild($parameterNode);
		}
		
		return $dsNode;
	}
	
	protected function createParameterNodes($dsId)
	{	
		curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json"));
		curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets/".$dsId."/variables");			   
			 
	 	$obj = json_decode(curl_exec($this->ch));
	 	$parameters = $obj->VariableDescription;
d4071e27   Elena.Budnik   makeProxy with en...
304

41792000   Elena.Budnik   new REMOTEDATA
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
		$paramNodes = array();
		
		foreach ($parameters as $param)
		{
			$paramNode = $this->domAmda->createElement('parameter');
			$paramNode->setAttribute("xml:id", $this->baseID.":".$dsId.":".$param->Name);
			$paramNode->setAttribute("name", $param->Name);
			
			$paramNodes[] = $paramNode;
		}	
		
		return $paramNodes;
	}
	
	protected function getDatasetSpaseDescription($dsID)
	{
	//	$this->openConnection();		 
		curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/xml"));

		if ( !array_key_exists($dsID, $this->CDAWEB ))
			return -1; // no description in SpaseRegistry
			
		curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$spaseResolver."id=".$this->CDAWEB[$dsID]);
155b6b3c   Elena.Budnik   other approach to...
328
329
330
331
332

		if (!$this->spase_res->loadXML(curl_exec($this->ch)))
			return -100;
 
		$messages = $this->spase_res->getElementsByTagName('Message');
41792000   Elena.Budnik   new REMOTEDATA
333
334
335
336
337
338
339
340
		
		if ($messages->length > 0)
		{
			foreach ($messages as $message)
					echo $message->nodeValue.PHP_EOL;
			return -2;   // no description in  www-spase
		}
		
155b6b3c   Elena.Budnik   other approach to...
341
342
// 		$instrument = $this->spase_res->getElementsByTagName('InstrumentID');
// 	   echo "     instrument ".$instrument->item(0)->nodeValue.PHP_EOL;
41792000   Elena.Budnik   new REMOTEDATA
343
		
155b6b3c   Elena.Budnik   other approach to...
344
		$cadence = $this->spase_res->getElementsByTagName('Cadence');
41792000   Elena.Budnik   new REMOTEDATA
345
		
155b6b3c   Elena.Budnik   other approach to...
346
		if ($cadence->length == 0)
41792000   Elena.Budnik   new REMOTEDATA
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
			return -3; // no cadence in spase xml
			
		$sampling = $this->cadence2sampling($cadence->item(0)->nodeValue);
		
		return $sampling; 	 
	}
	
	public function cadence2sampling($cadence)
	{
		$scale = array('S' => 1, 'M' => 60, 'H' => '3600');
	   
		$value = substr($cadence,2,strlen($cadence)-3);
 
		$units = substr($cadence,-1); 
	   
0fb01225   Elena.Budnik   float sampling
362
363
		$sampling = floatval($value)*$scale[$units];
		
41792000   Elena.Budnik   new REMOTEDATA
364
365
366
		return $sampling;
	}
	
d4071e27   Elena.Budnik   makeProxy with en...
367
	// Get IDs and SPASE URLs of all CDAWEB SPASE-defined datasets 
41792000   Elena.Budnik   new REMOTEDATA
368
369
370
	protected function getAllSpaseDatasets()
	{
		require_once "simple_html_dom.php";
873ce7b2   Elena.Budnik   CDAWEB exclude so...
371

8c0d5a9f   Elena.Budnik   correct location ...
372
373
		if (file_exists($this->location."/NumericalData.html")) 
				rename($this->location."/NumericalData.html",$this->location."/NumericalData.html.bak");
41792000   Elena.Budnik   new REMOTEDATA
374
		
d4071e27   Elena.Budnik   makeProxy with en...
375
		// if cannot reach CDAWEB Spase Registry use an old file
8c0d5a9f   Elena.Budnik   correct location ...
376
377
		if (!copy(CDAWebConfigClass::$spaseRegistry, $this->location."/NumericalData.html"))
				copy($this->location."/NumericalData.html.bak", $this->location."/NumericalData.html");
d4071e27   Elena.Budnik   makeProxy with en...
378

8c0d5a9f   Elena.Budnik   correct location ...
379
		$html = file_get_html($this->location.'/NumericalData.html');
41792000   Elena.Budnik   new REMOTEDATA
380
		$ids = $html->find('td[class="Spase.URL.ProductID"]');
873ce7b2   Elena.Budnik   CDAWEB exclude so...
381
		
41792000   Elena.Budnik   new REMOTEDATA
382
		foreach ($ids as $id)
873ce7b2   Elena.Budnik   CDAWEB exclude so...
383
384
385
		{	
		  
         if ($id->next_sibling()) {
41792000   Elena.Budnik   new REMOTEDATA
386
387
		   $ref = $id->next_sibling()->find('a');
		   $key = $id->find('a');
873ce7b2   Elena.Budnik   CDAWEB exclude so...
388
389
390
		   $this->CDAWEB[$key[0]->innertext] = $ref[0]->innertext;	
		   }
   
41792000   Elena.Budnik   new REMOTEDATA
391
		}
8c0d5a9f   Elena.Budnik   correct location ...
392
		error_log("SPASE-defined CDAWEB datasets from ".CDAWebConfigClass::$spaseRegistry." : ".count($this->CDAWEB).PHP_EOL,3,log);
05fbcab0   Elena.Budnik   DDServices reorga...
393
		echo "SPASE-defined CDAWEB datasets : ".count($this->CDAWEB).PHP_EOL;
41792000   Elena.Budnik   new REMOTEDATA
394
395
396
397
398
399
400
401
402
403
404
405
406
407
	}
	
	protected function initDDServerXml($ds, $ins, $obs)
	{
		$this->DDserverXml = new DomDocument("1.0");
		$this->DDserverXmlName =  $this->DDserverDir."/".$ds.".xml";
		$rootNode = $this->DDserverXml->createElement('VI');
		$this->DDserverXml->appendChild($rootNode);
		$rootNode->appendChild($this->DDserverXml->createElement("DataBaseID", $this->baseID));
		$rootNode->appendChild($this->DDserverXml->createElement("Mission", $obs));
		$rootNode->appendChild($this->DDserverXml->createElement("Instrument", $ins));
		$rootNode->appendChild($this->DDserverXml->createElement("DataSetID", $ds));
		$rootNode->appendChild($this->DDserverXml->createElement("GlobalStart"));
		$rootNode->appendChild($this->DDserverXml->createElement("GlobalStop"));
1b0e0631   Elena.Budnik   unique name for obs
408
409
		$rootNode->appendChild($this->DDserverXml->createElement("MinSampling"));
		// fill value by default - if different - add into param description
2f2f6cee   Elena.Budnik   FillValue by default
410
		$rootNode->appendChild($this->DDserverXml->createElement("FillValue", -1.0E31));
41792000   Elena.Budnik   new REMOTEDATA
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
	}
	
	protected function updateDDServerXml($target, $value)
	{
		$node = $this->DDserverXml->getElementsByTagName($target);
		$node->item(0)->nodeValue = $value;
	}
	
	protected function saveDDServerXml()
	{
		$this->DDserverXml->save($this->DDserverXmlName);
	}
	
	protected function existsMasterCdf($dsId)
	{
d1327dc0   Benjamin Renard   Fix RemoteData ac...
426
		$this->initStreamContext();
41792000   Elena.Budnik   new REMOTEDATA
427
428
429
430
431
		$file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf";
		$file_headers = @get_headers($file);
		
		if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
			$exists = false;
8c0d5a9f   Elena.Budnik   correct location ...
432
			error_log("INFO:   no master CDF ".$dsId.PHP_EOL,3,log);
41792000   Elena.Budnik   new REMOTEDATA
433
434
435
436
437
438
439
440
441
442
		}
		else {
			$exists = true;
		}
		
		return $exists;
	}
		 
	protected function getMasterCdf($dsId)
	{
d1327dc0   Benjamin Renard   Fix RemoteData ac...
443
		$this->initStreamContext();
41792000   Elena.Budnik   new REMOTEDATA
444
445
446
447
448
449
450
451
452
453
454
		$file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf";
		$file_headers = @get_headers($file);
		
		if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
			 return false;
		}		 
		return $file;		 	 
	}
	
	public function getData($ds, $start, $stop)
	{
d1327dc0   Benjamin Renard   Fix RemoteData ac...
455
		$this->initStreamContext();
41792000   Elena.Budnik   new REMOTEDATA
456
457
458
459
		$this->openConnection();
		$this->setDataViewURL(); 

	   curl_setopt($this->ch, CURLOPT_URL,$this->dataViewURL."/datasets/$ds/orig_data/$start,$stop/");
873ce7b2   Elena.Budnik   CDAWEB exclude so...
460
	
41792000   Elena.Budnik   new REMOTEDATA
461
	   $res = new DomDocument("1.0");
873ce7b2   Elena.Budnik   CDAWEB exclude so...
462
	    
41792000   Elena.Budnik   new REMOTEDATA
463
	   $res->loadXML(curl_exec($this->ch));
873ce7b2   Elena.Budnik   CDAWEB exclude so...
464
	   
41792000   Elena.Budnik   new REMOTEDATA
465
466
	   if ($res->getElementsByTagName("html")->length > 0)
	   {
8c0d5a9f   Elena.Budnik   correct location ...
467
468
			error_log("ERROR no response for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
			error_log($res->saveXML(),3,err);
14659123   Elena.Budnik   return empty arra...
469
470
			$this->closeConnection();
			return array();
41792000   Elena.Budnik   new REMOTEDATA
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
	   }
	   
	   $fileNames = $res->getElementsByTagName("Name");
	   	   
		$nc_prefix = strlen($ds) > RemoteDataCenterClass::$MAX_VI_NAME_LENGTH ? 
				substr(strtolower($ds),0,RemoteDataCenterClass::$MAX_VI_NAME_LENGTH - 1): strtolower($ds);
		$files = array();
	   
		for ($i = 0; $i < $fileNames->length;  $i++)
	   {
			$fileName = $fileNames->item($i);
			$url = $fileName->nodeValue;
			$file_headers = @get_headers($url);
		
			if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
8c0d5a9f   Elena.Budnik   correct location ...
486
				error_log("ERROR  404 Not Found for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
41792000   Elena.Budnik   new REMOTEDATA
487
488
489
490
				continue;
			}	
			$temp = explode('/',$url);
			$destination = $temp[count($temp)-1];
7216c535   Elena.Budnik   added samplingCDA...
491

41792000   Elena.Budnik   new REMOTEDATA
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
			if (copy($url, $destination))
			{
				$ncFile = $this->convert2nc($destination);
				if (strlen($ncFile) > RemoteDataCenterClass::$MAX_NAME_LENGTH)
				{
					$ncFileTrue = $nc_prefix."_".time().$i.".nc";
					rename($ncFile, $ncFileTrue);
					$files[] = $ncFileTrue;
				}
				else
				{
					$files[] = $ncFile;
				}
			}
			else
			{
8c0d5a9f   Elena.Budnik   correct location ...
508
				error_log("ERROR cannot copy files : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
41792000   Elena.Budnik   new REMOTEDATA
509
510
511
512
513
514
515
516
517
518
			}
	   }
	   
		$this->closeConnection();
		
		return $files;
	}
	
	protected function convert2nc($file)
	{
0fb01225   Elena.Budnik   float sampling
519
		system("cdfnew2nc $file");		 
41792000   Elena.Budnik   new REMOTEDATA
520
521
		$ncFile = str_replace(".cdf", ".nc", $file);
		
8c0d5a9f   Elena.Budnik   correct location ...
522
523
524
525
 		if (!file_exists($ncFile)) {
			error_log("ERROR while converting ".$file.PHP_EOL,3,err);
			return false; 
 		}
41792000   Elena.Budnik   new REMOTEDATA
526
527
528
529
530
		unlink($file);	
		
		return $ncFile;
	}
	
0ec21281   Elena.Budnik   reorganization + ...
531
532
	public function getDatasetInfo($ds)
	{
35322ea6   Elena.Budnik   case of absence o...
533
534
535
		if (!$this->existsMasterCdf($ds)) return false;
		
		$masterCdf = $this->getMasterCdf($ds);
0ec21281   Elena.Budnik   reorganization + ...
536
		$localCdf = strtolower($ds).".cdf";
35322ea6   Elena.Budnik   case of absence o...
537
		if (!copy($masterCdf, $localCdf)) return false;
0ec21281   Elena.Budnik   reorganization + ...
538
539
540
541
542
	 
		$infoFile = $this->convert2nc($localCdf);
	  
		return $infoFile;
	}
155b6b3c   Elena.Budnik   other approach to...
543
544
545
546
547
548
549
550
551
552
553
554
555
	
	public function makeProxy()
	{	 
		$this->spase_res = new DomDocument("1.0");
		$this->domAmda = new DOMDocument('1.0', 'utf-8');
		$this->domAmda->formatOutput = TRUE;
		$this->domAmda->preserveWhiteSpace = FALSE;
		
		$dataRoot = $this->domAmda->createElement('dataRoot');
		$dataRoot->setAttribute('xml:id', 'myRemoteData-treeRootNode');
		$this->domAmda->appendChild($dataRoot);	
		
		$this->dataCenter=$this->domAmda->createElement('dataCenter');
8c0d5a9f   Elena.Budnik   correct location ...
556
557
	 	$this->setDataCenterAttributes();
	 	
155b6b3c   Elena.Budnik   other approach to...
558
		$dataRoot->appendChild($this->dataCenter);
8c0d5a9f   Elena.Budnik   correct location ...
559
				
1b0e0631   Elena.Budnik   unique name for obs
560
561
		$first = true;
		
8c0d5a9f   Elena.Budnik   correct location ...
562
563
		foreach ($this->obsGroupsIds as $insType => $groupIds)
		{		
1b0e0631   Elena.Budnik   unique name for obs
564
565
566
567
568
			// getElementByID doesn't work on newly created Document
			if (!$first) {
				$this->domAmda->loadXML($this->domAmda->saveXML());			
				$this->dataCenter = $this->domAmda->getElementById($this->baseID);
			}
1b0e0631   Elena.Budnik   unique name for obs
569
			
8c0d5a9f   Elena.Budnik   correct location ...
570
571
			foreach ($groupIds as $groupId => $obss)
			{			
155b6b3c   Elena.Budnik   other approach to...
572
				$obsNodes = array();
775592ff   Elena.Budnik   bug in insNodes a...
573
				
1b0e0631   Elena.Budnik   unique name for obs
574
				foreach ($obss as $obs => $inss)
8c0d5a9f   Elena.Budnik   correct location ...
575
576
				{	
					// ! observatory OMNI is taken for observatoriesGroup OMNI only !
1b0e0631   Elena.Budnik   unique name for obs
577
578
579
					if (substr($obs,0,4) == "OMNI" && substr($groupId,0,4) != "OMNI"){
						echo 'OMNI      '.$obs.' '.$missionId.PHP_EOL;
						continue;
775592ff   Elena.Budnik   bug in insNodes a...
580
					} 
155b6b3c   Elena.Budnik   other approach to...
581
582
					$insNodes = array();
					// Create instrument nodes
8c0d5a9f   Elena.Budnik   correct location ...
583
584
					foreach ($inss as $ins => $dss)
					{
155b6b3c   Elena.Budnik   other approach to...
585
586
						$dsNodes = array();
						// Create datasets nodes
8c0d5a9f   Elena.Budnik   correct location ...
587
588
						foreach ($dss as $ds)
						{	
155b6b3c   Elena.Budnik   other approach to...
589
							$dsId = $this->baseID.":".$ds;
155b6b3c   Elena.Budnik   other approach to...
590
591
592
593
594
595
596
597
598
							if (!$this->domAmda->getElementById($dsId)){
								$this->initDDServerXml($ds,$ins,$obs);
								$dsNode = $this->createDatasetNode($ds);
								if ($dsNode){
									$this->saveDDServerXml();									
									$dsNodes[] = $dsNode;
								}
							}	
						} // foreach ($dss as $ds)
8c0d5a9f   Elena.Budnik   correct location ...
599
600
						if ($dsNodes)
						{
155b6b3c   Elena.Budnik   other approach to...
601
602
603
604
605
							// last $spase_res : instrument should be the same
							$insSpaseId = $this->getInstrumentSpase();
							if (!$insSpaseId ) 
								$insSpaseId = $ins;
								
775592ff   Elena.Budnik   bug in insNodes a...
606
							$obsSpaseId = strtolower($this->getObservatorySpase());
62372d94   Elena.Budnik   RemoveVi
607
							if (!$obsSpaseId ) 
775592ff   Elena.Budnik   bug in insNodes a...
608
								$obsSpaseId = strtolower($obs);
62372d94   Elena.Budnik   RemoveVi
609
610
							
							$insId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId).":".$this->param2dd($insSpaseId);
155b6b3c   Elena.Budnik   other approach to...
611
612
							
							if (!($insNode = $this->domAmda->getElementById($insId))){
62372d94   Elena.Budnik   RemoveVi
613
								$insNode = $this->createInstrumentNode($insSpaseId, $obsSpaseId, $groupId);
775592ff   Elena.Budnik   bug in insNodes a...
614
							} 
155b6b3c   Elena.Budnik   other approach to...
615
616
617
							foreach ($dsNodes as $dsNode){
								$insNode->appendChild($dsNode);
							}
775592ff   Elena.Budnik   bug in insNodes a...
618
							$insNodes[] = $insNode;
155b6b3c   Elena.Budnik   other approach to...
619
620
						}
					} // foreach ($inss as $ins => $dss)
8c0d5a9f   Elena.Budnik   correct location ...
621
622
					if ($insNodes)
					{					
62372d94   Elena.Budnik   RemoveVi
623
						$obsId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId);
155b6b3c   Elena.Budnik   other approach to...
624
						
1b0e0631   Elena.Budnik   unique name for obs
625
						if (!($obsNode = $this->domAmda->getElementById($obsId))){   
62372d94   Elena.Budnik   RemoveVi
626
							$obsNode = $this->createObservatoryNode($obsSpaseId, $groupId);
155b6b3c   Elena.Budnik   other approach to...
627
628
629
630
631
						}
						
						foreach ($insNodes as $insNode){
							$obsNode->appendChild($insNode);
						}
775592ff   Elena.Budnik   bug in insNodes a...
632
633
						
						$obsNodes[] = $obsNode;
155b6b3c   Elena.Budnik   other approach to...
634
635
					}
				} // foreach ($obss as $obs => $inss)
155b6b3c   Elena.Budnik   other approach to...
636
				
8c0d5a9f   Elena.Budnik   correct location ...
637
638
				if ($obsNodes)
				{
1b0e0631   Elena.Budnik   unique name for obs
639
640
641
642
643
					$missionId = $this->baseID.":".$this->param2dd($groupId);
					
					if (!($missionNode = $this->domAmda->getElementById($missionId))){
						$missionNode = $this->createMissionNode($groupId);
						$this->dataCenter->appendChild($missionNode);
62372d94   Elena.Budnik   RemoveVi
644
					}
1b0e0631   Elena.Budnik   unique name for obs
645
					
8c0d5a9f   Elena.Budnik   correct location ...
646
647
					foreach ($obsNodes as $obsNode)
					{
1b0e0631   Elena.Budnik   unique name for obs
648
						// observatory == mission
8c0d5a9f   Elena.Budnik   correct location ...
649
						if (strtolower($obsNode->getAttribute('name')) == strtolower($missionNode->getAttribute('name'))) {
1b0e0631   Elena.Budnik   unique name for obs
650
651
652
653
654
655
656
							$insNodes = $obsNode->getElementsByTagName("instrument");
							foreach ($insNodes as $insNode)
								$missionNode->appendChild($insNode);
						}
						else 
							$missionNode->appendChild($obsNode);
					}	
155b6b3c   Elena.Budnik   other approach to...
657
				}
1b0e0631   Elena.Budnik   unique name for obs
658
659
			}
			$first = false;
155b6b3c   Elena.Budnik   other approach to...
660
		}
05fbcab0   Elena.Budnik   DDServices reorga...
661
662
		
		$this->closeConnection();
155b6b3c   Elena.Budnik   other approach to...
663
664
	}
	
8c0d5a9f   Elena.Budnik   correct location ...
665
666
	protected function createMissionNode($groupId)
	{
155b6b3c   Elena.Budnik   other approach to...
667
668
669
670
671
672
673
674
		$missionId = $this->baseID.":".$this->param2dd($groupId);
		$missionNode = $this->domAmda->createElement('mission');
		$missionNode->setAttribute("xml:id",$missionId);
		$missionNode->setAttribute("name",$groupId);
		
		return $missionNode;
	}
	
8c0d5a9f   Elena.Budnik   correct location ...
675
676
	protected function getInstrumentSpase()
	{	
62372d94   Elena.Budnik   RemoveVi
677
		$this->insXML = new DomDocument("1.0");
155b6b3c   Elena.Budnik   other approach to...
678
679
680
681
682
		
		if ($this->spase_res){
			$instrument = $this->spase_res->getElementsByTagName('InstrumentID');
			
			if ($instrument->length > 0) {
62372d94   Elena.Budnik   RemoveVi
683
684
685
686
687
				
				curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$spaseResolver."id=".$instrument->item(0)->nodeValue);
				
				$this->insXML->loadXML(curl_exec($this->ch));
				
155b6b3c   Elena.Budnik   other approach to...
688
				return $this->getIdFromSpase($instrument->item(0)->nodeValue);
155b6b3c   Elena.Budnik   other approach to...
689
690
691
692
693
694
695
696
				//InstrumentType
				//spase://SMWG/Observatory/ACE
			} 
		}	
		
		return null;
	}
	
8c0d5a9f   Elena.Budnik   correct location ...
697
698
	protected function getObservatorySpase()
	{
62372d94   Elena.Budnik   RemoveVi
699
700
701
702
703
704
705
706
707
708
709
		if ($this->insXML){
			$observatory = $this->insXML->getElementsByTagName('ObservatoryID');
			
			if ($observatory->length > 0) {
				return $this->getIdFromSpase($observatory->item(0)->nodeValue); 			
			} 
		}	
		
		return null;
	}
	
8c0d5a9f   Elena.Budnik   correct location ...
710
711
712
713
714
715
	protected function setDataCenterAttributes(){	
		$this->dataCenter->setAttribute('xml:id', $this->baseID);
	//	$this->dataCenter->setIdAttribute('xml:id', true);
		$this->dataCenter->setAttribute('name', $this->baseID);		
	}
	
41792000   Elena.Budnik   new REMOTEDATA
716
	protected function makeArgumentsList(){}
05fbcab0   Elena.Budnik   DDServices reorga...
717

41792000   Elena.Budnik   new REMOTEDATA
718
719
} 
?>