Blame view

src/REMOTEDATA/CDAWEB.php 20.8 KB
41792000   Elena.Budnik   new REMOTEDATA
1
2
3
4
5
6
7
8
9
10
<?php
/**
 * @class CDAWEB 
 * @brief  CDAWeb 
 * @details
 */
class CDAWEB extends RemoteDataCenterClass
{
	private $ch, $res, $dataViewUR; 
	private $obsGroupsIds;
62372d94   Elena.Budnik   RemoveVi
11
	private $spase_res, $insXML;
41792000   Elena.Budnik   new REMOTEDATA
12
13
14
	
	private $observatoryGroups, $instrumentTypes;
	private $CDAWEB = array(), $masterCDF = array();
05fbcab0   Elena.Budnik   DDServices reorga...
15
	private $header;
41792000   Elena.Budnik   new REMOTEDATA
16
17
	
	// https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/instrumentTypes
1b0e0631   Elena.Budnik   unique name for obs
18
	private $validInstrumentTypes = [ "Activity%20Indices", "Electric%20Fields%20(space)","Gamma%20and%20X-Rays",
41792000   Elena.Budnik   new REMOTEDATA
19
                                     "Magnetic%20Fields%20(space)", "Particles%20(space)", "Plasma%20and%20Solar%20Wind", 
1b0e0631   Elena.Budnik   unique name for obs
20
                                     "Radio%20and%20Plasma%20Waves%20(space)" ];     
775592ff   Elena.Budnik   bug in insNodes a...
21
//    private $validInstrumentTypes = [ "Radio%20and%20Plasma%20Waves%20(space)" ];                               
155b6b3c   Elena.Budnik   other approach to...
22
                                     
41792000   Elena.Budnik   new REMOTEDATA
23
24
25
26
27
28
//   not used : $validInstrumentTypes = ["Ephemeris"];
    
	private $excludeIns = ["GIFWALK", "Unknown"], $excludeGroup = ["THEMIS", "ARTEMIS", "NOAA"];
	
	// not in "https://heliophysicsdata.sci.gsfc.nasa.gov/queries/CDAWeb_SPASE.xql" List;
	// FOR INFO : Excluded automatically
775592ff   Elena.Budnik   bug in insNodes a...
29
   private $excludeDatasets = [ "DMSP_R0_SSJ4", "DMSP_R0_SSIES", "I7_R0_LEPEDEA", "I8_R0_LEPEDEA" ];
41792000   Elena.Budnik   new REMOTEDATA
30
   
d4071e27   Elena.Budnik   makeProxy with en...
31
	protected $DDserverXml, $DDserverXmlName;
41792000   Elena.Budnik   new REMOTEDATA
32
	
41792000   Elena.Budnik   new REMOTEDATA
33
34
35
36
37
38
39
40
41
	
	public function html_encode($param) 
	{
		$pairs = array(" " => "%20"); 
		
		return strtr($param,$pairs); 
	}
	
	public function init() 	
8c0d5a9f   Elena.Budnik   correct location ...
42
	{			
35322ea6   Elena.Budnik   case of absence o...
43
44
		error_log("CDAWEB Proxy creation on ".date("Y-m-d\TH:i:s").PHP_EOL,3,log);
		error_log("CDAWEB Proxy creation on ".date("Y-m-d\TH:i:s").PHP_EOL,3,err);
d1327dc0   Benjamin Renard   Fix RemoteData ac...
45
46

		$this->initStreamContext();
05fbcab0   Elena.Budnik   DDServices reorga...
47
		
41792000   Elena.Budnik   new REMOTEDATA
48
49
50
		$this->getAllSpaseDatasets();		
	}			 
		
d1327dc0   Benjamin Renard   Fix RemoteData ac...
51
52
	private function openConnection() 
	{	
41792000   Elena.Budnik   new REMOTEDATA
53
54
		$this->ch = curl_init();			 
		curl_setopt($this->ch, CURLOPT_RETURNTRANSFER, 1);
d1327dc0   Benjamin Renard   Fix RemoteData ac...
55
56
57
58
59
60
61
62
63
64
		curl_setopt($this->ch, CURLOPT_TIMEOUT, 60);
		// Add proxy definition
		$PROXY_HOST=getenv('PROXY_HOST');
		$PROXY_USERPWD=getenv('PROXY_USERPWD');
		if (!empty($PROXY_HOST)) {
			curl_setopt($this->ch, CURLOPT_PROXY, getenv('PROXY_HOST'));
			if (!empty($PROXY_USERPWD)) {
				curl_setopt($this->ch, CURLOPT_PROXYUSERPWD, getenv('PROXY_USERPWD'));
			}
		}
41792000   Elena.Budnik   new REMOTEDATA
65
66
67
68
69
70
	} 
		  
	private function closeConnection()
	{
		curl_close($this->ch);	
	}
d1327dc0   Benjamin Renard   Fix RemoteData ac...
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

	private function initStreamContext()
	{
		$PROXY_HOST=getenv('PROXY_HOST');
		$PROXY_USERPWD=getenv('PROXY_USERPWD');
		if (!empty($PROXY_HOST)) {
			$context = array(
				'http' => array(
					'proxy' => "tcp://$PROXY_HOST",
					'request_fulluri' => true,
				),
			);
			if (!empty($PROXY_USERPWD)) {
				$context['http']['header'] = "Proxy-Authorization: Basic ".base64_encode($PROXY_USERPWD);
                        }
			stream_context_set_default($context);
		}
	}
41792000   Elena.Budnik   new REMOTEDATA
89
	
41792000   Elena.Budnik   new REMOTEDATA
90
91
92
	protected function setDataViewURL()
	{
		curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$restUrl."/dataviews");
d4071e27   Elena.Budnik   makeProxy with en...
93
		
41792000   Elena.Budnik   new REMOTEDATA
94
95
		$this->res = new DomDocument();	
		$this->res->loadXML(curl_exec($this->ch));
775592ff   Elena.Budnik   bug in insNodes a...
96
97
98
99
100
101
		
		if ($this->res->getElementsByTagName('EndpointAddress')->length == 0) {
				error_log("Problem connect to ".CDAWebConfigClass::$restUrl."/dataviews", 3, err);
				exit("Problem connect to ".CDAWebConfigClass::$restUrl."/dataviews".PHP_EOL);
		}
		
41792000   Elena.Budnik   new REMOTEDATA
102
103
		$this->dataViewURL = $this->res->getElementsByTagName('EndpointAddress')->item(0)->nodeValue;
	}
d4071e27   Elena.Budnik   makeProxy with en...
104
105
	
	// Using CDAS REST APIs get all CDAWEB dataset descriptions
41792000   Elena.Budnik   new REMOTEDATA
106
107
108
109
110
111
112
	protected function getRemoteTree()	
	{		
		$this->openConnection();
		
		$this->setDataViewURL();     
      $this->obsGroupsIds = array();           
           
775592ff   Elena.Budnik   bug in insNodes a...
113
      foreach ($this->validInstrumentTypes as $insType)
41792000   Elena.Budnik   new REMOTEDATA
114
115
116
117
      {		
			curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/observatoryGroupsAndInstruments?instrumentType=".$insType);			   
			echo $insType.PHP_EOL;
			$this->res->loadXML(curl_exec($this->ch));
775592ff   Elena.Budnik   bug in insNodes a...
118
119

			$observatoryGroups = $this->res->getElementsByTagName("ObservatoryGroupInstrumentDescription");
41792000   Elena.Budnik   new REMOTEDATA
120
 		   
775592ff   Elena.Budnik   bug in insNodes a...
121
122
			foreach ($observatoryGroups as $obsGroup)
			{
41792000   Elena.Budnik   new REMOTEDATA
123
124
125
126
127
128
129
130
131
132
				$obsGroupId = $obsGroup->getElementsByTagName('Name')->item(0)->nodeValue;
				
				// Exclude some Missions
				if (in_array ($obsGroupId, $this->excludeGroup )) continue;
				
				echo "   ".$obsGroupId.PHP_EOL;
				$observatories =  $obsGroup->getElementsByTagName("ObservatoryInstruments");
				if ( $observatories->length == 0 )
				{
					echo '      No observatories '.PHP_EOL;
8c0d5a9f   Elena.Budnik   correct location ...
133
134
135
136
					if ($obsGroup->getElementsByTagName("InstrumentDescription")->length > 0) {
							echo '      CHECK INSTRUMENTS '.PHP_EOL;
							error_log(" CHECK INSTRUMENTS ".$obsGroup.PHP_EOL, 3, err);
					}	
41792000   Elena.Budnik   new REMOTEDATA
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
					continue;
				}
				$obsIds = array();
				foreach ( $observatories as $obs )
				{
					$obsId = $obs->getElementsByTagName('Name')->item(0)->nodeValue;
					echo "      ".$obsId.PHP_EOL;
// 				<InstrumentDescription>
// 				  <Name>MAG</Name>
// 				  <ShortDescription>ACE Magnetic Field Instrument</ShortDescription>
// 				  <LongDescription>ACE Magnetic Field Instrument</LongDescription>
// 			   </InstrumentDescription>
					$instruments =  $obs->getElementsByTagName("InstrumentDescription");
					
					$insIds = array();
					
					foreach ( $instruments as $ins )
					{
						$insId = $ins->getElementsByTagName('Name')->item(0)->nodeValue;
						
						// Exclude some 'non-AMDA' staff
						if (in_array ($insId, $this->excludeIns )) continue;
						
						$dsIds = $this->getDatasetByObservatoryAndInstrument($obsGroupId, $obsId, $insId);
					   
					   $insIds[$insId] = $dsIds;
  
					}
					
					$obsIds[$obsId] = $insIds;
				}
				
1b0e0631   Elena.Budnik   unique name for obs
169
				$this->obsGroupsIds[$insType][$obsGroupId] = $obsIds;
41792000   Elena.Budnik   new REMOTEDATA
170
			}
05fbcab0   Elena.Budnik   DDServices reorga...
171
		}		
41792000   Elena.Budnik   new REMOTEDATA
172
173
174
	}
	
	protected function getDatasetByObservatoryAndInstrument($obsGroupId,  $obsId, $insId)
8c0d5a9f   Elena.Budnik   correct location ...
175
	{		
41792000   Elena.Budnik   new REMOTEDATA
176
177
		$grpEncodedId = $this->html_encode($obsGroupId);
		$insEncodedId = $this->html_encode($insId);
05fbcab0   Elena.Budnik   DDServices reorga...
178
		$obsEncodedId = $this->html_encode($obsId);
41792000   Elena.Budnik   new REMOTEDATA
179
180
		
		$res = new DomDocument("1.0");
05fbcab0   Elena.Budnik   DDServices reorga...
181
182
183
	//	curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets?observatoryGroup=$grpEncodedId&instrument=$insEncodedId");
	
		curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets?observatory=$obsEncodedId&instrument=$insEncodedId");
41792000   Elena.Budnik   new REMOTEDATA
184
185
		
		$res->loadXML(curl_exec($this->ch));
05fbcab0   Elena.Budnik   DDServices reorga...
186

41792000   Elena.Budnik   new REMOTEDATA
187
188
189
190
191
		$datasets = $res->getElementsByTagName('DatasetDescription');
		$dsIds = array();
		
		foreach ( $datasets as $ds)
		{
05fbcab0   Elena.Budnik   DDServices reorga...
192
		//	$obs = $ds->getElementsByTagName('Observatory')->item(0)->nodeValue;
8c0d5a9f   Elena.Budnik   correct location ...
193
		//	if ($obs != $obsId ) continue;
41792000   Elena.Budnik   new REMOTEDATA
194
195
196
197
198
199
			$dsIds[] = $ds->getElementsByTagName('Id')->item(0)->nodeValue;
		}
		
		return $dsIds;  
	}
	
41792000   Elena.Budnik   new REMOTEDATA
200
201
202
	protected function createObservatoryNode($id, $grpId)
	{	
	
155b6b3c   Elena.Budnik   other approach to...
203
		$obsNode = $this->domAmda->createElement('observatory');
41792000   Elena.Budnik   new REMOTEDATA
204
				
155b6b3c   Elena.Budnik   other approach to...
205
206
		$obsNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($grpId).":".$this->param2dd($id));
		$obsNode->setAttribute("name",$id);
41792000   Elena.Budnik   new REMOTEDATA
207
		
155b6b3c   Elena.Budnik   other approach to...
208
		return $obsNode;
41792000   Elena.Budnik   new REMOTEDATA
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
	}
	
	protected function createInstrumentNode($id, $obsId, $groupId)
	{	
	
		$insNode = $this->domAmda->createElement('instrument');
				
		$insNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsId).":".$id);
		$insNode->setAttribute("name",$id);
	  //	$insNode->setAttribute("description",$ins[1]);
		
		return $insNode;
	}
	
	protected function createDatasetNode($dsId)
	{		
		$dsNode = $this->domAmda->createElement('dataset');
		
		curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json"));
		curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets/?idPattern=".$dsId);			   
			
		$obj = json_decode(curl_exec($this->ch));
      $dataSet = $obj->DatasetDescription;
      
		$dsNode->setAttribute("xml:id",$this->baseID.":".$dsId);
		$dsNode->setAttribute("name", $dsId);
		$startTime = $dataSet[0]->TimeInterval->Start;
		$endTime =  $dataSet[0]->TimeInterval->End;
		$label = $dataSet[0]->Label; 
		$url = $dataSet[0]->DatasetLink[0]->Url;
		$notesUrl = $dataSet[0]->Notes; 
		$piAffilation = $dataSet[0]->PiAffiliation;
		$piName = $dataSet[0]->PiName; 
		$sampling = $this->getDatasetSpaseDescription($dsId);
		$this->updateDDServerXml("GlobalStart",$startTime);
		$this->updateDDServerXml("GlobalStop",$endTime);
1b0e0631   Elena.Budnik   unique name for obs
245
		
41792000   Elena.Budnik   new REMOTEDATA
246
		// no general description - strange dataset
8c0d5a9f   Elena.Budnik   correct location ...
247
248
		if ($sampling == -1) {
			error_log("! No general description : ".$dsId.PHP_EOL,3,err);
41792000   Elena.Budnik   new REMOTEDATA
249
			return null;
8c0d5a9f   Elena.Budnik   correct location ...
250
		}
41792000   Elena.Budnik   new REMOTEDATA
251
			
8c0d5a9f   Elena.Budnik   correct location ...
252
		if ($sampling < -1)	{
41792000   Elena.Budnik   new REMOTEDATA
253
			echo "       !!! $dsId $sampling".PHP_EOL;
8c0d5a9f   Elena.Budnik   correct location ...
254
255
			error_log("! No sampling : ".$dsId." : ".$sampling.PHP_EOL,3,err);
		}
41792000   Elena.Budnik   new REMOTEDATA
256
257
		else
			$this->updateDDServerXml("MinSampling",$sampling);	
1b0e0631   Elena.Budnik   unique name for obs
258
		
155b6b3c   Elena.Budnik   other approach to...
259
		$dsNode->setAttribute('spaseUrl',$this->CDAWEB[$dsId]);
41792000   Elena.Budnik   new REMOTEDATA
260
	//   $dsNode->setAttribute('masterCdf',$this->existsMasterCdf($dsId));
155b6b3c   Elena.Budnik   other approach to...
261
		$dsNode->setAttribute("desc", "$label; $startTime - $endTime");
21b939db   Elena.Budnik   dataStart, dataSt...
262
263
		$dsNode->setAttribute("dataStart", $startTime); 
		$dsNode->setAttribute("dataStop", $endTime); 
41792000   Elena.Budnik   new REMOTEDATA
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
	   
		$parameterNodes = $this->createParameterNodes($dsId);
		foreach ( $parameterNodes as $parameterNode)
		{
			$dsNode->appendChild($parameterNode);
		}
		
		return $dsNode;
	}
	
	protected function createParameterNodes($dsId)
	{	
		curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json"));
		curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets/".$dsId."/variables");			   
			 
	 	$obj = json_decode(curl_exec($this->ch));
	 	$parameters = $obj->VariableDescription;
d4071e27   Elena.Budnik   makeProxy with en...
281

41792000   Elena.Budnik   new REMOTEDATA
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
		$paramNodes = array();
		
		foreach ($parameters as $param)
		{
			$paramNode = $this->domAmda->createElement('parameter');
			$paramNode->setAttribute("xml:id", $this->baseID.":".$dsId.":".$param->Name);
			$paramNode->setAttribute("name", $param->Name);
			
			$paramNodes[] = $paramNode;
		}	
		
		return $paramNodes;
	}
	
	protected function getDatasetSpaseDescription($dsID)
	{
	//	$this->openConnection();		 
		curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/xml"));

		if ( !array_key_exists($dsID, $this->CDAWEB ))
			return -1; // no description in SpaseRegistry
			
		curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$spaseResolver."id=".$this->CDAWEB[$dsID]);
155b6b3c   Elena.Budnik   other approach to...
305
306
307
308
309

		if (!$this->spase_res->loadXML(curl_exec($this->ch)))
			return -100;
 
		$messages = $this->spase_res->getElementsByTagName('Message');
41792000   Elena.Budnik   new REMOTEDATA
310
311
312
313
314
315
316
317
		
		if ($messages->length > 0)
		{
			foreach ($messages as $message)
					echo $message->nodeValue.PHP_EOL;
			return -2;   // no description in  www-spase
		}
		
155b6b3c   Elena.Budnik   other approach to...
318
319
// 		$instrument = $this->spase_res->getElementsByTagName('InstrumentID');
// 	   echo "     instrument ".$instrument->item(0)->nodeValue.PHP_EOL;
41792000   Elena.Budnik   new REMOTEDATA
320
		
155b6b3c   Elena.Budnik   other approach to...
321
		$cadence = $this->spase_res->getElementsByTagName('Cadence');
41792000   Elena.Budnik   new REMOTEDATA
322
		
155b6b3c   Elena.Budnik   other approach to...
323
		if ($cadence->length == 0)
41792000   Elena.Budnik   new REMOTEDATA
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
			return -3; // no cadence in spase xml
			
		$sampling = $this->cadence2sampling($cadence->item(0)->nodeValue);
		
		return $sampling; 	 
	}
	
	public function cadence2sampling($cadence)
	{
		$scale = array('S' => 1, 'M' => 60, 'H' => '3600');
	   
		$value = substr($cadence,2,strlen($cadence)-3);
 
		$units = substr($cadence,-1); 
	   
0fb01225   Elena.Budnik   float sampling
339
340
		$sampling = floatval($value)*$scale[$units];
		
41792000   Elena.Budnik   new REMOTEDATA
341
342
343
		return $sampling;
	}
	
d4071e27   Elena.Budnik   makeProxy with en...
344
	// Get IDs and SPASE URLs of all CDAWEB SPASE-defined datasets 
41792000   Elena.Budnik   new REMOTEDATA
345
346
347
348
	protected function getAllSpaseDatasets()
	{
		require_once "simple_html_dom.php";
		
8c0d5a9f   Elena.Budnik   correct location ...
349
350
		if (file_exists($this->location."/NumericalData.html")) 
				rename($this->location."/NumericalData.html",$this->location."/NumericalData.html.bak");
41792000   Elena.Budnik   new REMOTEDATA
351
		
d4071e27   Elena.Budnik   makeProxy with en...
352
		// if cannot reach CDAWEB Spase Registry use an old file
8c0d5a9f   Elena.Budnik   correct location ...
353
354
		if (!copy(CDAWebConfigClass::$spaseRegistry, $this->location."/NumericalData.html"))
				copy($this->location."/NumericalData.html.bak", $this->location."/NumericalData.html");
d4071e27   Elena.Budnik   makeProxy with en...
355

8c0d5a9f   Elena.Budnik   correct location ...
356
		$html = file_get_html($this->location.'/NumericalData.html');
41792000   Elena.Budnik   new REMOTEDATA
357
358
359
360
361
362
363
364
		$ids = $html->find('td[class="Spase.URL.ProductID"]');

		foreach ($ids as $id)
		{		    
		   $ref = $id->next_sibling()->find('a');
		   $key = $id->find('a');
		   $this->CDAWEB[$key[0]->innertext] = $ref[0]->innertext;		   
		}
8c0d5a9f   Elena.Budnik   correct location ...
365
		error_log("SPASE-defined CDAWEB datasets from ".CDAWebConfigClass::$spaseRegistry." : ".count($this->CDAWEB).PHP_EOL,3,log);
05fbcab0   Elena.Budnik   DDServices reorga...
366
		echo "SPASE-defined CDAWEB datasets : ".count($this->CDAWEB).PHP_EOL;
41792000   Elena.Budnik   new REMOTEDATA
367
368
369
370
371
372
373
374
375
376
377
378
379
380
	}
	
	protected function initDDServerXml($ds, $ins, $obs)
	{
		$this->DDserverXml = new DomDocument("1.0");
		$this->DDserverXmlName =  $this->DDserverDir."/".$ds.".xml";
		$rootNode = $this->DDserverXml->createElement('VI');
		$this->DDserverXml->appendChild($rootNode);
		$rootNode->appendChild($this->DDserverXml->createElement("DataBaseID", $this->baseID));
		$rootNode->appendChild($this->DDserverXml->createElement("Mission", $obs));
		$rootNode->appendChild($this->DDserverXml->createElement("Instrument", $ins));
		$rootNode->appendChild($this->DDserverXml->createElement("DataSetID", $ds));
		$rootNode->appendChild($this->DDserverXml->createElement("GlobalStart"));
		$rootNode->appendChild($this->DDserverXml->createElement("GlobalStop"));
1b0e0631   Elena.Budnik   unique name for obs
381
382
		$rootNode->appendChild($this->DDserverXml->createElement("MinSampling"));
		// fill value by default - if different - add into param description
2f2f6cee   Elena.Budnik   FillValue by default
383
		$rootNode->appendChild($this->DDserverXml->createElement("FillValue", -1.0E31));
41792000   Elena.Budnik   new REMOTEDATA
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
	}
	
	protected function updateDDServerXml($target, $value)
	{
		$node = $this->DDserverXml->getElementsByTagName($target);
		$node->item(0)->nodeValue = $value;
	}
	
	protected function saveDDServerXml()
	{
		$this->DDserverXml->save($this->DDserverXmlName);
	}
	
	protected function existsMasterCdf($dsId)
	{
d1327dc0   Benjamin Renard   Fix RemoteData ac...
399
		$this->initStreamContext();
41792000   Elena.Budnik   new REMOTEDATA
400
401
402
403
404
		$file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf";
		$file_headers = @get_headers($file);
		
		if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
			$exists = false;
8c0d5a9f   Elena.Budnik   correct location ...
405
			error_log("INFO:   no master CDF ".$dsId.PHP_EOL,3,log);
41792000   Elena.Budnik   new REMOTEDATA
406
407
408
409
410
411
412
413
414
415
		}
		else {
			$exists = true;
		}
		
		return $exists;
	}
		 
	protected function getMasterCdf($dsId)
	{
d1327dc0   Benjamin Renard   Fix RemoteData ac...
416
		$this->initStreamContext();
41792000   Elena.Budnik   new REMOTEDATA
417
418
419
420
421
422
423
424
425
426
427
		$file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf";
		$file_headers = @get_headers($file);
		
		if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
			 return false;
		}		 
		return $file;		 	 
	}
	
	public function getData($ds, $start, $stop)
	{
d1327dc0   Benjamin Renard   Fix RemoteData ac...
428
		$this->initStreamContext();
41792000   Elena.Budnik   new REMOTEDATA
429
430
431
432
433
434
435
		$this->openConnection();
		$this->setDataViewURL(); 

	   curl_setopt($this->ch, CURLOPT_URL,$this->dataViewURL."/datasets/$ds/orig_data/$start,$stop/");
	   $res = new DomDocument("1.0");
	   
	   $res->loadXML(curl_exec($this->ch));
8c0d5a9f   Elena.Budnik   correct location ...
436

41792000   Elena.Budnik   new REMOTEDATA
437
438
	   if ($res->getElementsByTagName("html")->length > 0)
	   {
8c0d5a9f   Elena.Budnik   correct location ...
439
440
			error_log("ERROR no response for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
			error_log($res->saveXML(),3,err);
41792000   Elena.Budnik   new REMOTEDATA
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
			exit();
	   }
	   
	   $fileNames = $res->getElementsByTagName("Name");
	   	   
		$nc_prefix = strlen($ds) > RemoteDataCenterClass::$MAX_VI_NAME_LENGTH ? 
				substr(strtolower($ds),0,RemoteDataCenterClass::$MAX_VI_NAME_LENGTH - 1): strtolower($ds);
		$files = array();
	   
		for ($i = 0; $i < $fileNames->length;  $i++)
	   {
			$fileName = $fileNames->item($i);
			$url = $fileName->nodeValue;
			$file_headers = @get_headers($url);
		
			if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
8c0d5a9f   Elena.Budnik   correct location ...
457
				error_log("ERROR  404 Not Found for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
41792000   Elena.Budnik   new REMOTEDATA
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
				continue;
			}	
			$temp = explode('/',$url);
			$destination = $temp[count($temp)-1];
			
			if (copy($url, $destination))
			{
				$ncFile = $this->convert2nc($destination);
				if (strlen($ncFile) > RemoteDataCenterClass::$MAX_NAME_LENGTH)
				{
					$ncFileTrue = $nc_prefix."_".time().$i.".nc";
					rename($ncFile, $ncFileTrue);
					$files[] = $ncFileTrue;
				}
				else
				{
					$files[] = $ncFile;
				}
			}
			else
			{
8c0d5a9f   Elena.Budnik   correct location ...
479
				error_log("ERROR cannot copy files : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
41792000   Elena.Budnik   new REMOTEDATA
480
481
482
483
484
485
486
487
488
489
			}
	   }
	   
		$this->closeConnection();
		
		return $files;
	}
	
	protected function convert2nc($file)
	{
0fb01225   Elena.Budnik   float sampling
490
		system("cdfnew2nc $file");		 
41792000   Elena.Budnik   new REMOTEDATA
491
492
		$ncFile = str_replace(".cdf", ".nc", $file);
		
8c0d5a9f   Elena.Budnik   correct location ...
493
494
495
496
 		if (!file_exists($ncFile)) {
			error_log("ERROR while converting ".$file.PHP_EOL,3,err);
			return false; 
 		}
41792000   Elena.Budnik   new REMOTEDATA
497
498
499
500
501
		unlink($file);	
		
		return $ncFile;
	}
	
0ec21281   Elena.Budnik   reorganization + ...
502
503
	public function getDatasetInfo($ds)
	{
35322ea6   Elena.Budnik   case of absence o...
504
505
506
		if (!$this->existsMasterCdf($ds)) return false;
		
		$masterCdf = $this->getMasterCdf($ds);
0ec21281   Elena.Budnik   reorganization + ...
507
508
		$localCdf = strtolower($ds).".cdf";
		
35322ea6   Elena.Budnik   case of absence o...
509
		if (!copy($masterCdf, $localCdf)) return false;
0ec21281   Elena.Budnik   reorganization + ...
510
511
512
513
514
	 
		$infoFile = $this->convert2nc($localCdf);
	  
		return $infoFile;
	}
155b6b3c   Elena.Budnik   other approach to...
515
516
517
518
519
520
521
522
523
524
525
526
527
	
	public function makeProxy()
	{	 
		$this->spase_res = new DomDocument("1.0");
		$this->domAmda = new DOMDocument('1.0', 'utf-8');
		$this->domAmda->formatOutput = TRUE;
		$this->domAmda->preserveWhiteSpace = FALSE;
		
		$dataRoot = $this->domAmda->createElement('dataRoot');
		$dataRoot->setAttribute('xml:id', 'myRemoteData-treeRootNode');
		$this->domAmda->appendChild($dataRoot);	
		
		$this->dataCenter=$this->domAmda->createElement('dataCenter');
8c0d5a9f   Elena.Budnik   correct location ...
528
529
	 	$this->setDataCenterAttributes();
	 	
155b6b3c   Elena.Budnik   other approach to...
530
		$dataRoot->appendChild($this->dataCenter);
8c0d5a9f   Elena.Budnik   correct location ...
531
				
1b0e0631   Elena.Budnik   unique name for obs
532
533
		$first = true;
		
8c0d5a9f   Elena.Budnik   correct location ...
534
535
		foreach ($this->obsGroupsIds as $insType => $groupIds)
		{		
1b0e0631   Elena.Budnik   unique name for obs
536
537
538
539
540
			// getElementByID doesn't work on newly created Document
			if (!$first) {
				$this->domAmda->loadXML($this->domAmda->saveXML());			
				$this->dataCenter = $this->domAmda->getElementById($this->baseID);
			}
1b0e0631   Elena.Budnik   unique name for obs
541
			
8c0d5a9f   Elena.Budnik   correct location ...
542
543
			foreach ($groupIds as $groupId => $obss)
			{			
155b6b3c   Elena.Budnik   other approach to...
544
				$obsNodes = array();
775592ff   Elena.Budnik   bug in insNodes a...
545
				
1b0e0631   Elena.Budnik   unique name for obs
546
				foreach ($obss as $obs => $inss)
8c0d5a9f   Elena.Budnik   correct location ...
547
548
				{	
					// ! observatory OMNI is taken for observatoriesGroup OMNI only !
1b0e0631   Elena.Budnik   unique name for obs
549
550
551
					if (substr($obs,0,4) == "OMNI" && substr($groupId,0,4) != "OMNI"){
						echo 'OMNI      '.$obs.' '.$missionId.PHP_EOL;
						continue;
775592ff   Elena.Budnik   bug in insNodes a...
552
					} 
155b6b3c   Elena.Budnik   other approach to...
553
554
					$insNodes = array();
					// Create instrument nodes
8c0d5a9f   Elena.Budnik   correct location ...
555
556
					foreach ($inss as $ins => $dss)
					{
155b6b3c   Elena.Budnik   other approach to...
557
558
						$dsNodes = array();
						// Create datasets nodes
8c0d5a9f   Elena.Budnik   correct location ...
559
560
						foreach ($dss as $ds)
						{	
155b6b3c   Elena.Budnik   other approach to...
561
							$dsId = $this->baseID.":".$ds;
155b6b3c   Elena.Budnik   other approach to...
562
563
564
565
566
567
568
569
570
							if (!$this->domAmda->getElementById($dsId)){
								$this->initDDServerXml($ds,$ins,$obs);
								$dsNode = $this->createDatasetNode($ds);
								if ($dsNode){
									$this->saveDDServerXml();									
									$dsNodes[] = $dsNode;
								}
							}	
						} // foreach ($dss as $ds)
8c0d5a9f   Elena.Budnik   correct location ...
571
572
						if ($dsNodes)
						{
155b6b3c   Elena.Budnik   other approach to...
573
574
575
576
577
							// last $spase_res : instrument should be the same
							$insSpaseId = $this->getInstrumentSpase();
							if (!$insSpaseId ) 
								$insSpaseId = $ins;
								
775592ff   Elena.Budnik   bug in insNodes a...
578
							$obsSpaseId = strtolower($this->getObservatorySpase());
62372d94   Elena.Budnik   RemoveVi
579
							if (!$obsSpaseId ) 
775592ff   Elena.Budnik   bug in insNodes a...
580
								$obsSpaseId = strtolower($obs);
62372d94   Elena.Budnik   RemoveVi
581
582
							
							$insId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId).":".$this->param2dd($insSpaseId);
155b6b3c   Elena.Budnik   other approach to...
583
584
							
							if (!($insNode = $this->domAmda->getElementById($insId))){
62372d94   Elena.Budnik   RemoveVi
585
								$insNode = $this->createInstrumentNode($insSpaseId, $obsSpaseId, $groupId);
775592ff   Elena.Budnik   bug in insNodes a...
586
							} 
155b6b3c   Elena.Budnik   other approach to...
587
588
589
							foreach ($dsNodes as $dsNode){
								$insNode->appendChild($dsNode);
							}
775592ff   Elena.Budnik   bug in insNodes a...
590
							$insNodes[] = $insNode;
155b6b3c   Elena.Budnik   other approach to...
591
592
						}
					} // foreach ($inss as $ins => $dss)
8c0d5a9f   Elena.Budnik   correct location ...
593
594
					if ($insNodes)
					{					
62372d94   Elena.Budnik   RemoveVi
595
						$obsId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId);
155b6b3c   Elena.Budnik   other approach to...
596
						
1b0e0631   Elena.Budnik   unique name for obs
597
						if (!($obsNode = $this->domAmda->getElementById($obsId))){   
62372d94   Elena.Budnik   RemoveVi
598
							$obsNode = $this->createObservatoryNode($obsSpaseId, $groupId);
155b6b3c   Elena.Budnik   other approach to...
599
600
601
602
603
						}
						
						foreach ($insNodes as $insNode){
							$obsNode->appendChild($insNode);
						}
775592ff   Elena.Budnik   bug in insNodes a...
604
605
						
						$obsNodes[] = $obsNode;
155b6b3c   Elena.Budnik   other approach to...
606
607
					}
				} // foreach ($obss as $obs => $inss)
155b6b3c   Elena.Budnik   other approach to...
608
				
8c0d5a9f   Elena.Budnik   correct location ...
609
610
				if ($obsNodes)
				{
1b0e0631   Elena.Budnik   unique name for obs
611
612
613
614
615
					$missionId = $this->baseID.":".$this->param2dd($groupId);
					
					if (!($missionNode = $this->domAmda->getElementById($missionId))){
						$missionNode = $this->createMissionNode($groupId);
						$this->dataCenter->appendChild($missionNode);
62372d94   Elena.Budnik   RemoveVi
616
					}
1b0e0631   Elena.Budnik   unique name for obs
617
					
8c0d5a9f   Elena.Budnik   correct location ...
618
619
					foreach ($obsNodes as $obsNode)
					{
1b0e0631   Elena.Budnik   unique name for obs
620
						// observatory == mission
8c0d5a9f   Elena.Budnik   correct location ...
621
						if (strtolower($obsNode->getAttribute('name')) == strtolower($missionNode->getAttribute('name'))) {
1b0e0631   Elena.Budnik   unique name for obs
622
623
624
625
626
627
628
							$insNodes = $obsNode->getElementsByTagName("instrument");
							foreach ($insNodes as $insNode)
								$missionNode->appendChild($insNode);
						}
						else 
							$missionNode->appendChild($obsNode);
					}	
155b6b3c   Elena.Budnik   other approach to...
629
				}
1b0e0631   Elena.Budnik   unique name for obs
630
631
			}
			$first = false;
155b6b3c   Elena.Budnik   other approach to...
632
		}
05fbcab0   Elena.Budnik   DDServices reorga...
633
634
		
		$this->closeConnection();
155b6b3c   Elena.Budnik   other approach to...
635
636
	}
	
8c0d5a9f   Elena.Budnik   correct location ...
637
638
	protected function createMissionNode($groupId)
	{
155b6b3c   Elena.Budnik   other approach to...
639
640
641
642
643
644
645
646
		$missionId = $this->baseID.":".$this->param2dd($groupId);
		$missionNode = $this->domAmda->createElement('mission');
		$missionNode->setAttribute("xml:id",$missionId);
		$missionNode->setAttribute("name",$groupId);
		
		return $missionNode;
	}
	
8c0d5a9f   Elena.Budnik   correct location ...
647
648
	protected function getInstrumentSpase()
	{	
62372d94   Elena.Budnik   RemoveVi
649
		$this->insXML = new DomDocument("1.0");
155b6b3c   Elena.Budnik   other approach to...
650
651
652
653
654
		
		if ($this->spase_res){
			$instrument = $this->spase_res->getElementsByTagName('InstrumentID');
			
			if ($instrument->length > 0) {
62372d94   Elena.Budnik   RemoveVi
655
656
657
658
659
				
				curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$spaseResolver."id=".$instrument->item(0)->nodeValue);
				
				$this->insXML->loadXML(curl_exec($this->ch));
				
155b6b3c   Elena.Budnik   other approach to...
660
				return $this->getIdFromSpase($instrument->item(0)->nodeValue);
155b6b3c   Elena.Budnik   other approach to...
661
662
663
664
665
666
667
668
				//InstrumentType
				//spase://SMWG/Observatory/ACE
			} 
		}	
		
		return null;
	}
	
8c0d5a9f   Elena.Budnik   correct location ...
669
670
	protected function getObservatorySpase()
	{
62372d94   Elena.Budnik   RemoveVi
671
672
673
674
675
676
677
678
679
680
681
		if ($this->insXML){
			$observatory = $this->insXML->getElementsByTagName('ObservatoryID');
			
			if ($observatory->length > 0) {
				return $this->getIdFromSpase($observatory->item(0)->nodeValue); 			
			} 
		}	
		
		return null;
	}
	
8c0d5a9f   Elena.Budnik   correct location ...
682
683
684
685
686
687
	protected function setDataCenterAttributes(){	
		$this->dataCenter->setAttribute('xml:id', $this->baseID);
	//	$this->dataCenter->setIdAttribute('xml:id', true);
		$this->dataCenter->setAttribute('name', $this->baseID);		
	}
	
41792000   Elena.Budnik   new REMOTEDATA
688
	protected function makeArgumentsList(){}
05fbcab0   Elena.Budnik   DDServices reorga...
689

41792000   Elena.Budnik   new REMOTEDATA
690
691
} 
?>