Blame view

src/REMOTEDATA/CDAWEB.php 21.1 KB
41792000   Elena.Budnik   new REMOTEDATA
1
2
3
4
5
6
7
8
9
10
<?php
/**
 * @class CDAWEB 
 * @brief  CDAWeb 
 * @details
 */
class CDAWEB extends RemoteDataCenterClass
{
	private $ch, $res, $dataViewUR; 
	private $obsGroupsIds;
62372d94   Elena.Budnik   RemoveVi
11
	private $spase_res, $insXML;
41792000   Elena.Budnik   new REMOTEDATA
12
13
14
	
	private $observatoryGroups, $instrumentTypes;
	private $CDAWEB = array(), $masterCDF = array();
05fbcab0   Elena.Budnik   DDServices reorga...
15
	private $header;
41792000   Elena.Budnik   new REMOTEDATA
16
17
	
	// https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/instrumentTypes
1b0e0631   Elena.Budnik   unique name for obs
18
	private $validInstrumentTypes = [ "Activity%20Indices", "Electric%20Fields%20(space)","Gamma%20and%20X-Rays",
41792000   Elena.Budnik   new REMOTEDATA
19
                                     "Magnetic%20Fields%20(space)", "Particles%20(space)", "Plasma%20and%20Solar%20Wind", 
1b0e0631   Elena.Budnik   unique name for obs
20
                                     "Radio%20and%20Plasma%20Waves%20(space)" ];     
775592ff   Elena.Budnik   bug in insNodes a...
21
//    private $validInstrumentTypes = [ "Radio%20and%20Plasma%20Waves%20(space)" ];                               
155b6b3c   Elena.Budnik   other approach to...
22
                                     
41792000   Elena.Budnik   new REMOTEDATA
23
24
//   not used : $validInstrumentTypes = ["Ephemeris"];
    
873ce7b2   Elena.Budnik   CDAWEB exclude so...
25
	private $excludeIns = ["GIFWALK", "Unknown"], $excludeGroup = ["THEMIS", "ARTEMIS", "NOAA", "GPS", "TSS-1R"];
41792000   Elena.Budnik   new REMOTEDATA
26
27
28
	
	// not in "https://heliophysicsdata.sci.gsfc.nasa.gov/queries/CDAWeb_SPASE.xql" List;
	// FOR INFO : Excluded automatically
775592ff   Elena.Budnik   bug in insNodes a...
29
   private $excludeDatasets = [ "DMSP_R0_SSJ4", "DMSP_R0_SSIES", "I7_R0_LEPEDEA", "I8_R0_LEPEDEA" ];
41792000   Elena.Budnik   new REMOTEDATA
30
   
d4071e27   Elena.Budnik   makeProxy with en...
31
	protected $DDserverXml, $DDserverXmlName;
41792000   Elena.Budnik   new REMOTEDATA
32
	
41792000   Elena.Budnik   new REMOTEDATA
33
34
35
36
37
38
39
40
41
	
	public function html_encode($param) 
	{
		$pairs = array(" " => "%20"); 
		
		return strtr($param,$pairs); 
	}
	
	public function init() 	
8c0d5a9f   Elena.Budnik   correct location ...
42
	{			
35322ea6   Elena.Budnik   case of absence o...
43
44
		error_log("CDAWEB Proxy creation on ".date("Y-m-d\TH:i:s").PHP_EOL,3,log);
		error_log("CDAWEB Proxy creation on ".date("Y-m-d\TH:i:s").PHP_EOL,3,err);
d1327dc0   Benjamin Renard   Fix RemoteData ac...
45
46

		$this->initStreamContext();
05fbcab0   Elena.Budnik   DDServices reorga...
47
		
41792000   Elena.Budnik   new REMOTEDATA
48
49
50
		$this->getAllSpaseDatasets();		
	}			 
		
d1327dc0   Benjamin Renard   Fix RemoteData ac...
51
52
	private function openConnection() 
	{	
41792000   Elena.Budnik   new REMOTEDATA
53
54
		$this->ch = curl_init();			 
		curl_setopt($this->ch, CURLOPT_RETURNTRANSFER, 1);
d1327dc0   Benjamin Renard   Fix RemoteData ac...
55
56
57
58
59
60
61
62
63
64
		curl_setopt($this->ch, CURLOPT_TIMEOUT, 60);
		// Add proxy definition
		$PROXY_HOST=getenv('PROXY_HOST');
		$PROXY_USERPWD=getenv('PROXY_USERPWD');
		if (!empty($PROXY_HOST)) {
			curl_setopt($this->ch, CURLOPT_PROXY, getenv('PROXY_HOST'));
			if (!empty($PROXY_USERPWD)) {
				curl_setopt($this->ch, CURLOPT_PROXYUSERPWD, getenv('PROXY_USERPWD'));
			}
		}
41792000   Elena.Budnik   new REMOTEDATA
65
66
67
68
69
70
	} 
		  
	private function closeConnection()
	{
		curl_close($this->ch);	
	}
d1327dc0   Benjamin Renard   Fix RemoteData ac...
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88

	private function initStreamContext()
	{
		$PROXY_HOST=getenv('PROXY_HOST');
		$PROXY_USERPWD=getenv('PROXY_USERPWD');
		if (!empty($PROXY_HOST)) {
			$context = array(
				'http' => array(
					'proxy' => "tcp://$PROXY_HOST",
					'request_fulluri' => true,
				),
			);
			if (!empty($PROXY_USERPWD)) {
				$context['http']['header'] = "Proxy-Authorization: Basic ".base64_encode($PROXY_USERPWD);
                        }
			stream_context_set_default($context);
		}
	}
41792000   Elena.Budnik   new REMOTEDATA
89
	
41792000   Elena.Budnik   new REMOTEDATA
90
91
92
	protected function setDataViewURL()
	{
		curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$restUrl."/dataviews");
d4071e27   Elena.Budnik   makeProxy with en...
93
		
41792000   Elena.Budnik   new REMOTEDATA
94
95
		$this->res = new DomDocument();	
		$this->res->loadXML(curl_exec($this->ch));
775592ff   Elena.Budnik   bug in insNodes a...
96
97
98
99
100
101
		
		if ($this->res->getElementsByTagName('EndpointAddress')->length == 0) {
				error_log("Problem connect to ".CDAWebConfigClass::$restUrl."/dataviews", 3, err);
				exit("Problem connect to ".CDAWebConfigClass::$restUrl."/dataviews".PHP_EOL);
		}
		
41792000   Elena.Budnik   new REMOTEDATA
102
103
		$this->dataViewURL = $this->res->getElementsByTagName('EndpointAddress')->item(0)->nodeValue;
	}
d4071e27   Elena.Budnik   makeProxy with en...
104
105
	
	// Using CDAS REST APIs get all CDAWEB dataset descriptions
41792000   Elena.Budnik   new REMOTEDATA
106
107
108
109
110
111
112
	protected function getRemoteTree()	
	{		
		$this->openConnection();
		
		$this->setDataViewURL();     
      $this->obsGroupsIds = array();           
           
775592ff   Elena.Budnik   bug in insNodes a...
113
      foreach ($this->validInstrumentTypes as $insType)
41792000   Elena.Budnik   new REMOTEDATA
114
115
116
117
      {		
			curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/observatoryGroupsAndInstruments?instrumentType=".$insType);			   
			echo $insType.PHP_EOL;
			$this->res->loadXML(curl_exec($this->ch));
775592ff   Elena.Budnik   bug in insNodes a...
118
119

			$observatoryGroups = $this->res->getElementsByTagName("ObservatoryGroupInstrumentDescription");
41792000   Elena.Budnik   new REMOTEDATA
120
 		   
775592ff   Elena.Budnik   bug in insNodes a...
121
122
			foreach ($observatoryGroups as $obsGroup)
			{
41792000   Elena.Budnik   new REMOTEDATA
123
124
125
126
127
128
129
130
131
132
				$obsGroupId = $obsGroup->getElementsByTagName('Name')->item(0)->nodeValue;
				
				// Exclude some Missions
				if (in_array ($obsGroupId, $this->excludeGroup )) continue;
				
				echo "   ".$obsGroupId.PHP_EOL;
				$observatories =  $obsGroup->getElementsByTagName("ObservatoryInstruments");
				if ( $observatories->length == 0 )
				{
					echo '      No observatories '.PHP_EOL;
8c0d5a9f   Elena.Budnik   correct location ...
133
134
135
136
					if ($obsGroup->getElementsByTagName("InstrumentDescription")->length > 0) {
							echo '      CHECK INSTRUMENTS '.PHP_EOL;
							error_log(" CHECK INSTRUMENTS ".$obsGroup.PHP_EOL, 3, err);
					}	
41792000   Elena.Budnik   new REMOTEDATA
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
					continue;
				}
				$obsIds = array();
				foreach ( $observatories as $obs )
				{
					$obsId = $obs->getElementsByTagName('Name')->item(0)->nodeValue;
					echo "      ".$obsId.PHP_EOL;
// 				<InstrumentDescription>
// 				  <Name>MAG</Name>
// 				  <ShortDescription>ACE Magnetic Field Instrument</ShortDescription>
// 				  <LongDescription>ACE Magnetic Field Instrument</LongDescription>
// 			   </InstrumentDescription>
					$instruments =  $obs->getElementsByTagName("InstrumentDescription");
					
					$insIds = array();
					
					foreach ( $instruments as $ins )
					{
						$insId = $ins->getElementsByTagName('Name')->item(0)->nodeValue;
						
						// Exclude some 'non-AMDA' staff
						if (in_array ($insId, $this->excludeIns )) continue;
						
						$dsIds = $this->getDatasetByObservatoryAndInstrument($obsGroupId, $obsId, $insId);
					   
					   $insIds[$insId] = $dsIds;
  
					}
					
					$obsIds[$obsId] = $insIds;
				}
				
1b0e0631   Elena.Budnik   unique name for obs
169
				$this->obsGroupsIds[$insType][$obsGroupId] = $obsIds;
41792000   Elena.Budnik   new REMOTEDATA
170
			}
05fbcab0   Elena.Budnik   DDServices reorga...
171
		}		
41792000   Elena.Budnik   new REMOTEDATA
172
173
174
	}
	
	protected function getDatasetByObservatoryAndInstrument($obsGroupId,  $obsId, $insId)
8c0d5a9f   Elena.Budnik   correct location ...
175
	{		
41792000   Elena.Budnik   new REMOTEDATA
176
177
		$grpEncodedId = $this->html_encode($obsGroupId);
		$insEncodedId = $this->html_encode($insId);
05fbcab0   Elena.Budnik   DDServices reorga...
178
		$obsEncodedId = $this->html_encode($obsId);
41792000   Elena.Budnik   new REMOTEDATA
179
180
		
		$res = new DomDocument("1.0");
05fbcab0   Elena.Budnik   DDServices reorga...
181
182
183
	//	curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets?observatoryGroup=$grpEncodedId&instrument=$insEncodedId");
	
		curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets?observatory=$obsEncodedId&instrument=$insEncodedId");
41792000   Elena.Budnik   new REMOTEDATA
184
185
		
		$res->loadXML(curl_exec($this->ch));
05fbcab0   Elena.Budnik   DDServices reorga...
186

41792000   Elena.Budnik   new REMOTEDATA
187
188
189
190
191
		$datasets = $res->getElementsByTagName('DatasetDescription');
		$dsIds = array();
		
		foreach ( $datasets as $ds)
		{
05fbcab0   Elena.Budnik   DDServices reorga...
192
		//	$obs = $ds->getElementsByTagName('Observatory')->item(0)->nodeValue;
8c0d5a9f   Elena.Budnik   correct location ...
193
		//	if ($obs != $obsId ) continue;
41792000   Elena.Budnik   new REMOTEDATA
194
195
196
197
198
199
			$dsIds[] = $ds->getElementsByTagName('Id')->item(0)->nodeValue;
		}
		
		return $dsIds;  
	}
	
41792000   Elena.Budnik   new REMOTEDATA
200
201
202
	protected function createObservatoryNode($id, $grpId)
	{	
	
155b6b3c   Elena.Budnik   other approach to...
203
		$obsNode = $this->domAmda->createElement('observatory');
41792000   Elena.Budnik   new REMOTEDATA
204
				
155b6b3c   Elena.Budnik   other approach to...
205
206
		$obsNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($grpId).":".$this->param2dd($id));
		$obsNode->setAttribute("name",$id);
41792000   Elena.Budnik   new REMOTEDATA
207
		
155b6b3c   Elena.Budnik   other approach to...
208
		return $obsNode;
41792000   Elena.Budnik   new REMOTEDATA
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
	}
	
	protected function createInstrumentNode($id, $obsId, $groupId)
	{	
	
		$insNode = $this->domAmda->createElement('instrument');
				
		$insNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsId).":".$id);
		$insNode->setAttribute("name",$id);
	  //	$insNode->setAttribute("description",$ins[1]);
		
		return $insNode;
	}
	
	protected function createDatasetNode($dsId)
	{		
		$dsNode = $this->domAmda->createElement('dataset');
		
		curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json"));
873ce7b2   Elena.Budnik   CDAWEB exclude so...
228
		curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets?idPattern=".$dsId);			   
41792000   Elena.Budnik   new REMOTEDATA
229
230
			
		$obj = json_decode(curl_exec($this->ch));
873ce7b2   Elena.Budnik   CDAWEB exclude so...
231
        $dataSet = $obj->DatasetDescription;
41792000   Elena.Budnik   new REMOTEDATA
232
233
234
235
236
237
238
239
240
241
242
243
244
      
		$dsNode->setAttribute("xml:id",$this->baseID.":".$dsId);
		$dsNode->setAttribute("name", $dsId);
		$startTime = $dataSet[0]->TimeInterval->Start;
		$endTime =  $dataSet[0]->TimeInterval->End;
		$label = $dataSet[0]->Label; 
		$url = $dataSet[0]->DatasetLink[0]->Url;
		$notesUrl = $dataSet[0]->Notes; 
		$piAffilation = $dataSet[0]->PiAffiliation;
		$piName = $dataSet[0]->PiName; 
		$sampling = $this->getDatasetSpaseDescription($dsId);
		$this->updateDDServerXml("GlobalStart",$startTime);
		$this->updateDDServerXml("GlobalStop",$endTime);
1b0e0631   Elena.Budnik   unique name for obs
245
		
41792000   Elena.Budnik   new REMOTEDATA
246
		// no general description - strange dataset
8c0d5a9f   Elena.Budnik   correct location ...
247
248
		if ($sampling == -1) {
			error_log("! No general description : ".$dsId.PHP_EOL,3,err);
41792000   Elena.Budnik   new REMOTEDATA
249
			return null;
8c0d5a9f   Elena.Budnik   correct location ...
250
		}
41792000   Elena.Budnik   new REMOTEDATA
251
			
8c0d5a9f   Elena.Budnik   correct location ...
252
		if ($sampling < -1)	{
873ce7b2   Elena.Budnik   CDAWEB exclude so...
253
			echo "       !!! $dsId sampling = $sampling".PHP_EOL;
8c0d5a9f   Elena.Budnik   correct location ...
254
255
			error_log("! No sampling : ".$dsId." : ".$sampling.PHP_EOL,3,err);
		}
873ce7b2   Elena.Budnik   CDAWEB exclude so...
256
257
258
259
260
		else if ($sampling == 0) {
            echo "       !!! $dsId sampling = $sampling".PHP_EOL;
			error_log("! No Sampling : ".$dsId." : ".$sampling.PHP_EOL,3,err);
        }
        else 
41792000   Elena.Budnik   new REMOTEDATA
261
			$this->updateDDServerXml("MinSampling",$sampling);	
1b0e0631   Elena.Budnik   unique name for obs
262
		
155b6b3c   Elena.Budnik   other approach to...
263
		$dsNode->setAttribute('spaseUrl',$this->CDAWEB[$dsId]);
41792000   Elena.Budnik   new REMOTEDATA
264
	//   $dsNode->setAttribute('masterCdf',$this->existsMasterCdf($dsId));
155b6b3c   Elena.Budnik   other approach to...
265
		$dsNode->setAttribute("desc", "$label; $startTime - $endTime");
21b939db   Elena.Budnik   dataStart, dataSt...
266
267
		$dsNode->setAttribute("dataStart", $startTime); 
		$dsNode->setAttribute("dataStop", $endTime); 
41792000   Elena.Budnik   new REMOTEDATA
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
	   
		$parameterNodes = $this->createParameterNodes($dsId);
		foreach ( $parameterNodes as $parameterNode)
		{
			$dsNode->appendChild($parameterNode);
		}
		
		return $dsNode;
	}
	
	protected function createParameterNodes($dsId)
	{	
		curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json"));
		curl_setopt($this->ch, CURLOPT_URL, $this->dataViewURL."/datasets/".$dsId."/variables");			   
			 
	 	$obj = json_decode(curl_exec($this->ch));
	 	$parameters = $obj->VariableDescription;
d4071e27   Elena.Budnik   makeProxy with en...
285

41792000   Elena.Budnik   new REMOTEDATA
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
		$paramNodes = array();
		
		foreach ($parameters as $param)
		{
			$paramNode = $this->domAmda->createElement('parameter');
			$paramNode->setAttribute("xml:id", $this->baseID.":".$dsId.":".$param->Name);
			$paramNode->setAttribute("name", $param->Name);
			
			$paramNodes[] = $paramNode;
		}	
		
		return $paramNodes;
	}
	
	protected function getDatasetSpaseDescription($dsID)
	{
	//	$this->openConnection();		 
		curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/xml"));

		if ( !array_key_exists($dsID, $this->CDAWEB ))
			return -1; // no description in SpaseRegistry
			
		curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$spaseResolver."id=".$this->CDAWEB[$dsID]);
155b6b3c   Elena.Budnik   other approach to...
309
310
311
312
313

		if (!$this->spase_res->loadXML(curl_exec($this->ch)))
			return -100;
 
		$messages = $this->spase_res->getElementsByTagName('Message');
41792000   Elena.Budnik   new REMOTEDATA
314
315
316
317
318
319
320
321
		
		if ($messages->length > 0)
		{
			foreach ($messages as $message)
					echo $message->nodeValue.PHP_EOL;
			return -2;   // no description in  www-spase
		}
		
155b6b3c   Elena.Budnik   other approach to...
322
323
// 		$instrument = $this->spase_res->getElementsByTagName('InstrumentID');
// 	   echo "     instrument ".$instrument->item(0)->nodeValue.PHP_EOL;
41792000   Elena.Budnik   new REMOTEDATA
324
		
155b6b3c   Elena.Budnik   other approach to...
325
		$cadence = $this->spase_res->getElementsByTagName('Cadence');
41792000   Elena.Budnik   new REMOTEDATA
326
		
155b6b3c   Elena.Budnik   other approach to...
327
		if ($cadence->length == 0)
41792000   Elena.Budnik   new REMOTEDATA
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
			return -3; // no cadence in spase xml
			
		$sampling = $this->cadence2sampling($cadence->item(0)->nodeValue);
		
		return $sampling; 	 
	}
	
	public function cadence2sampling($cadence)
	{
		$scale = array('S' => 1, 'M' => 60, 'H' => '3600');
	   
		$value = substr($cadence,2,strlen($cadence)-3);
 
		$units = substr($cadence,-1); 
	   
0fb01225   Elena.Budnik   float sampling
343
344
		$sampling = floatval($value)*$scale[$units];
		
41792000   Elena.Budnik   new REMOTEDATA
345
346
347
		return $sampling;
	}
	
d4071e27   Elena.Budnik   makeProxy with en...
348
	// Get IDs and SPASE URLs of all CDAWEB SPASE-defined datasets 
41792000   Elena.Budnik   new REMOTEDATA
349
350
351
	protected function getAllSpaseDatasets()
	{
		require_once "simple_html_dom.php";
873ce7b2   Elena.Budnik   CDAWEB exclude so...
352

8c0d5a9f   Elena.Budnik   correct location ...
353
354
		if (file_exists($this->location."/NumericalData.html")) 
				rename($this->location."/NumericalData.html",$this->location."/NumericalData.html.bak");
41792000   Elena.Budnik   new REMOTEDATA
355
		
d4071e27   Elena.Budnik   makeProxy with en...
356
		// if cannot reach CDAWEB Spase Registry use an old file
8c0d5a9f   Elena.Budnik   correct location ...
357
358
		if (!copy(CDAWebConfigClass::$spaseRegistry, $this->location."/NumericalData.html"))
				copy($this->location."/NumericalData.html.bak", $this->location."/NumericalData.html");
d4071e27   Elena.Budnik   makeProxy with en...
359

8c0d5a9f   Elena.Budnik   correct location ...
360
		$html = file_get_html($this->location.'/NumericalData.html');
41792000   Elena.Budnik   new REMOTEDATA
361
		$ids = $html->find('td[class="Spase.URL.ProductID"]');
873ce7b2   Elena.Budnik   CDAWEB exclude so...
362
		
41792000   Elena.Budnik   new REMOTEDATA
363
		foreach ($ids as $id)
873ce7b2   Elena.Budnik   CDAWEB exclude so...
364
365
366
		{	
		  
         if ($id->next_sibling()) {
41792000   Elena.Budnik   new REMOTEDATA
367
368
		   $ref = $id->next_sibling()->find('a');
		   $key = $id->find('a');
873ce7b2   Elena.Budnik   CDAWEB exclude so...
369
370
371
		   $this->CDAWEB[$key[0]->innertext] = $ref[0]->innertext;	
		   }
   
41792000   Elena.Budnik   new REMOTEDATA
372
		}
8c0d5a9f   Elena.Budnik   correct location ...
373
		error_log("SPASE-defined CDAWEB datasets from ".CDAWebConfigClass::$spaseRegistry." : ".count($this->CDAWEB).PHP_EOL,3,log);
05fbcab0   Elena.Budnik   DDServices reorga...
374
		echo "SPASE-defined CDAWEB datasets : ".count($this->CDAWEB).PHP_EOL;
41792000   Elena.Budnik   new REMOTEDATA
375
376
377
378
379
380
381
382
383
384
385
386
387
388
	}
	
	protected function initDDServerXml($ds, $ins, $obs)
	{
		$this->DDserverXml = new DomDocument("1.0");
		$this->DDserverXmlName =  $this->DDserverDir."/".$ds.".xml";
		$rootNode = $this->DDserverXml->createElement('VI');
		$this->DDserverXml->appendChild($rootNode);
		$rootNode->appendChild($this->DDserverXml->createElement("DataBaseID", $this->baseID));
		$rootNode->appendChild($this->DDserverXml->createElement("Mission", $obs));
		$rootNode->appendChild($this->DDserverXml->createElement("Instrument", $ins));
		$rootNode->appendChild($this->DDserverXml->createElement("DataSetID", $ds));
		$rootNode->appendChild($this->DDserverXml->createElement("GlobalStart"));
		$rootNode->appendChild($this->DDserverXml->createElement("GlobalStop"));
1b0e0631   Elena.Budnik   unique name for obs
389
390
		$rootNode->appendChild($this->DDserverXml->createElement("MinSampling"));
		// fill value by default - if different - add into param description
2f2f6cee   Elena.Budnik   FillValue by default
391
		$rootNode->appendChild($this->DDserverXml->createElement("FillValue", -1.0E31));
41792000   Elena.Budnik   new REMOTEDATA
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
	}
	
	protected function updateDDServerXml($target, $value)
	{
		$node = $this->DDserverXml->getElementsByTagName($target);
		$node->item(0)->nodeValue = $value;
	}
	
	protected function saveDDServerXml()
	{
		$this->DDserverXml->save($this->DDserverXmlName);
	}
	
	protected function existsMasterCdf($dsId)
	{
d1327dc0   Benjamin Renard   Fix RemoteData ac...
407
		$this->initStreamContext();
41792000   Elena.Budnik   new REMOTEDATA
408
409
410
411
412
		$file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf";
		$file_headers = @get_headers($file);
		
		if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
			$exists = false;
8c0d5a9f   Elena.Budnik   correct location ...
413
			error_log("INFO:   no master CDF ".$dsId.PHP_EOL,3,log);
41792000   Elena.Budnik   new REMOTEDATA
414
415
416
417
418
419
420
421
422
423
		}
		else {
			$exists = true;
		}
		
		return $exists;
	}
		 
	protected function getMasterCdf($dsId)
	{
d1327dc0   Benjamin Renard   Fix RemoteData ac...
424
		$this->initStreamContext();
41792000   Elena.Budnik   new REMOTEDATA
425
426
427
428
429
430
431
432
433
434
435
		$file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf";
		$file_headers = @get_headers($file);
		
		if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
			 return false;
		}		 
		return $file;		 	 
	}
	
	public function getData($ds, $start, $stop)
	{
d1327dc0   Benjamin Renard   Fix RemoteData ac...
436
		$this->initStreamContext();
41792000   Elena.Budnik   new REMOTEDATA
437
438
439
440
		$this->openConnection();
		$this->setDataViewURL(); 

	   curl_setopt($this->ch, CURLOPT_URL,$this->dataViewURL."/datasets/$ds/orig_data/$start,$stop/");
873ce7b2   Elena.Budnik   CDAWEB exclude so...
441
	
41792000   Elena.Budnik   new REMOTEDATA
442
	   $res = new DomDocument("1.0");
873ce7b2   Elena.Budnik   CDAWEB exclude so...
443
	    
41792000   Elena.Budnik   new REMOTEDATA
444
	   $res->loadXML(curl_exec($this->ch));
873ce7b2   Elena.Budnik   CDAWEB exclude so...
445
	   
41792000   Elena.Budnik   new REMOTEDATA
446
447
	   if ($res->getElementsByTagName("html")->length > 0)
	   {
8c0d5a9f   Elena.Budnik   correct location ...
448
449
			error_log("ERROR no response for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
			error_log($res->saveXML(),3,err);
14659123   Elena.Budnik   return empty arra...
450
451
			$this->closeConnection();
			return array();
41792000   Elena.Budnik   new REMOTEDATA
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
	   }
	   
	   $fileNames = $res->getElementsByTagName("Name");
	   	   
		$nc_prefix = strlen($ds) > RemoteDataCenterClass::$MAX_VI_NAME_LENGTH ? 
				substr(strtolower($ds),0,RemoteDataCenterClass::$MAX_VI_NAME_LENGTH - 1): strtolower($ds);
		$files = array();
	   
		for ($i = 0; $i < $fileNames->length;  $i++)
	   {
			$fileName = $fileNames->item($i);
			$url = $fileName->nodeValue;
			$file_headers = @get_headers($url);
		
			if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
8c0d5a9f   Elena.Budnik   correct location ...
467
				error_log("ERROR  404 Not Found for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
41792000   Elena.Budnik   new REMOTEDATA
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
				continue;
			}	
			$temp = explode('/',$url);
			$destination = $temp[count($temp)-1];
			
			if (copy($url, $destination))
			{
				$ncFile = $this->convert2nc($destination);
				if (strlen($ncFile) > RemoteDataCenterClass::$MAX_NAME_LENGTH)
				{
					$ncFileTrue = $nc_prefix."_".time().$i.".nc";
					rename($ncFile, $ncFileTrue);
					$files[] = $ncFileTrue;
				}
				else
				{
					$files[] = $ncFile;
				}
			}
			else
			{
8c0d5a9f   Elena.Budnik   correct location ...
489
				error_log("ERROR cannot copy files : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
41792000   Elena.Budnik   new REMOTEDATA
490
491
492
493
494
495
496
497
498
499
			}
	   }
	   
		$this->closeConnection();
		
		return $files;
	}
	
	protected function convert2nc($file)
	{
0fb01225   Elena.Budnik   float sampling
500
		system("cdfnew2nc $file");		 
41792000   Elena.Budnik   new REMOTEDATA
501
502
		$ncFile = str_replace(".cdf", ".nc", $file);
		
8c0d5a9f   Elena.Budnik   correct location ...
503
504
505
506
 		if (!file_exists($ncFile)) {
			error_log("ERROR while converting ".$file.PHP_EOL,3,err);
			return false; 
 		}
41792000   Elena.Budnik   new REMOTEDATA
507
508
509
510
511
		unlink($file);	
		
		return $ncFile;
	}
	
0ec21281   Elena.Budnik   reorganization + ...
512
513
	public function getDatasetInfo($ds)
	{
35322ea6   Elena.Budnik   case of absence o...
514
515
516
		if (!$this->existsMasterCdf($ds)) return false;
		
		$masterCdf = $this->getMasterCdf($ds);
0ec21281   Elena.Budnik   reorganization + ...
517
		$localCdf = strtolower($ds).".cdf";
35322ea6   Elena.Budnik   case of absence o...
518
		if (!copy($masterCdf, $localCdf)) return false;
0ec21281   Elena.Budnik   reorganization + ...
519
520
521
522
523
	 
		$infoFile = $this->convert2nc($localCdf);
	  
		return $infoFile;
	}
155b6b3c   Elena.Budnik   other approach to...
524
525
526
527
528
529
530
531
532
533
534
535
536
	
	public function makeProxy()
	{	 
		$this->spase_res = new DomDocument("1.0");
		$this->domAmda = new DOMDocument('1.0', 'utf-8');
		$this->domAmda->formatOutput = TRUE;
		$this->domAmda->preserveWhiteSpace = FALSE;
		
		$dataRoot = $this->domAmda->createElement('dataRoot');
		$dataRoot->setAttribute('xml:id', 'myRemoteData-treeRootNode');
		$this->domAmda->appendChild($dataRoot);	
		
		$this->dataCenter=$this->domAmda->createElement('dataCenter');
8c0d5a9f   Elena.Budnik   correct location ...
537
538
	 	$this->setDataCenterAttributes();
	 	
155b6b3c   Elena.Budnik   other approach to...
539
		$dataRoot->appendChild($this->dataCenter);
8c0d5a9f   Elena.Budnik   correct location ...
540
				
1b0e0631   Elena.Budnik   unique name for obs
541
542
		$first = true;
		
8c0d5a9f   Elena.Budnik   correct location ...
543
544
		foreach ($this->obsGroupsIds as $insType => $groupIds)
		{		
1b0e0631   Elena.Budnik   unique name for obs
545
546
547
548
549
			// getElementByID doesn't work on newly created Document
			if (!$first) {
				$this->domAmda->loadXML($this->domAmda->saveXML());			
				$this->dataCenter = $this->domAmda->getElementById($this->baseID);
			}
1b0e0631   Elena.Budnik   unique name for obs
550
			
8c0d5a9f   Elena.Budnik   correct location ...
551
552
			foreach ($groupIds as $groupId => $obss)
			{			
155b6b3c   Elena.Budnik   other approach to...
553
				$obsNodes = array();
775592ff   Elena.Budnik   bug in insNodes a...
554
				
1b0e0631   Elena.Budnik   unique name for obs
555
				foreach ($obss as $obs => $inss)
8c0d5a9f   Elena.Budnik   correct location ...
556
557
				{	
					// ! observatory OMNI is taken for observatoriesGroup OMNI only !
1b0e0631   Elena.Budnik   unique name for obs
558
559
560
					if (substr($obs,0,4) == "OMNI" && substr($groupId,0,4) != "OMNI"){
						echo 'OMNI      '.$obs.' '.$missionId.PHP_EOL;
						continue;
775592ff   Elena.Budnik   bug in insNodes a...
561
					} 
155b6b3c   Elena.Budnik   other approach to...
562
563
					$insNodes = array();
					// Create instrument nodes
8c0d5a9f   Elena.Budnik   correct location ...
564
565
					foreach ($inss as $ins => $dss)
					{
155b6b3c   Elena.Budnik   other approach to...
566
567
						$dsNodes = array();
						// Create datasets nodes
8c0d5a9f   Elena.Budnik   correct location ...
568
569
						foreach ($dss as $ds)
						{	
155b6b3c   Elena.Budnik   other approach to...
570
							$dsId = $this->baseID.":".$ds;
155b6b3c   Elena.Budnik   other approach to...
571
572
573
574
575
576
577
578
579
							if (!$this->domAmda->getElementById($dsId)){
								$this->initDDServerXml($ds,$ins,$obs);
								$dsNode = $this->createDatasetNode($ds);
								if ($dsNode){
									$this->saveDDServerXml();									
									$dsNodes[] = $dsNode;
								}
							}	
						} // foreach ($dss as $ds)
8c0d5a9f   Elena.Budnik   correct location ...
580
581
						if ($dsNodes)
						{
155b6b3c   Elena.Budnik   other approach to...
582
583
584
585
586
							// last $spase_res : instrument should be the same
							$insSpaseId = $this->getInstrumentSpase();
							if (!$insSpaseId ) 
								$insSpaseId = $ins;
								
775592ff   Elena.Budnik   bug in insNodes a...
587
							$obsSpaseId = strtolower($this->getObservatorySpase());
62372d94   Elena.Budnik   RemoveVi
588
							if (!$obsSpaseId ) 
775592ff   Elena.Budnik   bug in insNodes a...
589
								$obsSpaseId = strtolower($obs);
62372d94   Elena.Budnik   RemoveVi
590
591
							
							$insId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId).":".$this->param2dd($insSpaseId);
155b6b3c   Elena.Budnik   other approach to...
592
593
							
							if (!($insNode = $this->domAmda->getElementById($insId))){
62372d94   Elena.Budnik   RemoveVi
594
								$insNode = $this->createInstrumentNode($insSpaseId, $obsSpaseId, $groupId);
775592ff   Elena.Budnik   bug in insNodes a...
595
							} 
155b6b3c   Elena.Budnik   other approach to...
596
597
598
							foreach ($dsNodes as $dsNode){
								$insNode->appendChild($dsNode);
							}
775592ff   Elena.Budnik   bug in insNodes a...
599
							$insNodes[] = $insNode;
155b6b3c   Elena.Budnik   other approach to...
600
601
						}
					} // foreach ($inss as $ins => $dss)
8c0d5a9f   Elena.Budnik   correct location ...
602
603
					if ($insNodes)
					{					
62372d94   Elena.Budnik   RemoveVi
604
						$obsId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId);
155b6b3c   Elena.Budnik   other approach to...
605
						
1b0e0631   Elena.Budnik   unique name for obs
606
						if (!($obsNode = $this->domAmda->getElementById($obsId))){   
62372d94   Elena.Budnik   RemoveVi
607
							$obsNode = $this->createObservatoryNode($obsSpaseId, $groupId);
155b6b3c   Elena.Budnik   other approach to...
608
609
610
611
612
						}
						
						foreach ($insNodes as $insNode){
							$obsNode->appendChild($insNode);
						}
775592ff   Elena.Budnik   bug in insNodes a...
613
614
						
						$obsNodes[] = $obsNode;
155b6b3c   Elena.Budnik   other approach to...
615
616
					}
				} // foreach ($obss as $obs => $inss)
155b6b3c   Elena.Budnik   other approach to...
617
				
8c0d5a9f   Elena.Budnik   correct location ...
618
619
				if ($obsNodes)
				{
1b0e0631   Elena.Budnik   unique name for obs
620
621
622
623
624
					$missionId = $this->baseID.":".$this->param2dd($groupId);
					
					if (!($missionNode = $this->domAmda->getElementById($missionId))){
						$missionNode = $this->createMissionNode($groupId);
						$this->dataCenter->appendChild($missionNode);
62372d94   Elena.Budnik   RemoveVi
625
					}
1b0e0631   Elena.Budnik   unique name for obs
626
					
8c0d5a9f   Elena.Budnik   correct location ...
627
628
					foreach ($obsNodes as $obsNode)
					{
1b0e0631   Elena.Budnik   unique name for obs
629
						// observatory == mission
8c0d5a9f   Elena.Budnik   correct location ...
630
						if (strtolower($obsNode->getAttribute('name')) == strtolower($missionNode->getAttribute('name'))) {
1b0e0631   Elena.Budnik   unique name for obs
631
632
633
634
635
636
637
							$insNodes = $obsNode->getElementsByTagName("instrument");
							foreach ($insNodes as $insNode)
								$missionNode->appendChild($insNode);
						}
						else 
							$missionNode->appendChild($obsNode);
					}	
155b6b3c   Elena.Budnik   other approach to...
638
				}
1b0e0631   Elena.Budnik   unique name for obs
639
640
			}
			$first = false;
155b6b3c   Elena.Budnik   other approach to...
641
		}
05fbcab0   Elena.Budnik   DDServices reorga...
642
643
		
		$this->closeConnection();
155b6b3c   Elena.Budnik   other approach to...
644
645
	}
	
8c0d5a9f   Elena.Budnik   correct location ...
646
647
	protected function createMissionNode($groupId)
	{
155b6b3c   Elena.Budnik   other approach to...
648
649
650
651
652
653
654
655
		$missionId = $this->baseID.":".$this->param2dd($groupId);
		$missionNode = $this->domAmda->createElement('mission');
		$missionNode->setAttribute("xml:id",$missionId);
		$missionNode->setAttribute("name",$groupId);
		
		return $missionNode;
	}
	
8c0d5a9f   Elena.Budnik   correct location ...
656
657
	protected function getInstrumentSpase()
	{	
62372d94   Elena.Budnik   RemoveVi
658
		$this->insXML = new DomDocument("1.0");
155b6b3c   Elena.Budnik   other approach to...
659
660
661
662
663
		
		if ($this->spase_res){
			$instrument = $this->spase_res->getElementsByTagName('InstrumentID');
			
			if ($instrument->length > 0) {
62372d94   Elena.Budnik   RemoveVi
664
665
666
667
668
				
				curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$spaseResolver."id=".$instrument->item(0)->nodeValue);
				
				$this->insXML->loadXML(curl_exec($this->ch));
				
155b6b3c   Elena.Budnik   other approach to...
669
				return $this->getIdFromSpase($instrument->item(0)->nodeValue);
155b6b3c   Elena.Budnik   other approach to...
670
671
672
673
674
675
676
677
				//InstrumentType
				//spase://SMWG/Observatory/ACE
			} 
		}	
		
		return null;
	}
	
8c0d5a9f   Elena.Budnik   correct location ...
678
679
	protected function getObservatorySpase()
	{
62372d94   Elena.Budnik   RemoveVi
680
681
682
683
684
685
686
687
688
689
690
		if ($this->insXML){
			$observatory = $this->insXML->getElementsByTagName('ObservatoryID');
			
			if ($observatory->length > 0) {
				return $this->getIdFromSpase($observatory->item(0)->nodeValue); 			
			} 
		}	
		
		return null;
	}
	
8c0d5a9f   Elena.Budnik   correct location ...
691
692
693
694
695
696
	protected function setDataCenterAttributes(){	
		$this->dataCenter->setAttribute('xml:id', $this->baseID);
	//	$this->dataCenter->setIdAttribute('xml:id', true);
		$this->dataCenter->setAttribute('name', $this->baseID);		
	}
	
41792000   Elena.Budnik   new REMOTEDATA
697
	protected function makeArgumentsList(){}
05fbcab0   Elena.Budnik   DDServices reorga...
698

41792000   Elena.Budnik   new REMOTEDATA
699
700
} 
?>