Blame view

src/REMOTEDATA/CDAWEB.php 24.3 KB
41792000   Elena.Budnik   new REMOTEDATA
1
2
3
4
5
6
7
8
<?php
/**
 * @class CDAWEB 
 * @brief  CDAWeb 
 * @details
 */
class CDAWEB extends RemoteDataCenterClass
{
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
9
10
11
	private $dataViewURL = NULL;

	private $ch; 
41792000   Elena.Budnik   new REMOTEDATA
12
	private $obsGroupsIds;
7216c535   Elena.Budnik   added samplingCDA...
13
	private $spase_res, $insXML, $xp = null;
41792000   Elena.Budnik   new REMOTEDATA
14
15
16
	
	private $observatoryGroups, $instrumentTypes;
	private $CDAWEB = array(), $masterCDF = array();
05fbcab0   Elena.Budnik   DDServices reorga...
17
	private $header;
41792000   Elena.Budnik   new REMOTEDATA
18
19
	
	// https://cdaweb.gsfc.nasa.gov/WS/cdasr/1/dataviews/sp_phys/instrumentTypes
1b0e0631   Elena.Budnik   unique name for obs
20
	private $validInstrumentTypes = [ "Activity%20Indices", "Electric%20Fields%20(space)","Gamma%20and%20X-Rays",
41792000   Elena.Budnik   new REMOTEDATA
21
                                     "Magnetic%20Fields%20(space)", "Particles%20(space)", "Plasma%20and%20Solar%20Wind", 
217c83f2   Elena.Budnik   correct CDAWEB tr...
22
23
24
                                     "Radio%20and%20Plasma%20Waves%20(space)"]; //, "Ephemeris/Attitude/Ancillary" ];  

	private $excludeIns = ["GIFWALK", "Unknown"], $excludeGroup = ["THEMIS", "ARTEMIS", "MMS", "Apollo"]; // "NOAA", "GPS", "TSS-1R", "IBEX"];
41792000   Elena.Budnik   new REMOTEDATA
25
26
27
	
	// not in "https://heliophysicsdata.sci.gsfc.nasa.gov/queries/CDAWeb_SPASE.xql" List;
	// FOR INFO : Excluded automatically
7216c535   Elena.Budnik   added samplingCDA...
28
    private $excludeDatasets = [ "DMSP_R0_SSJ4", "DMSP_R0_SSIES", "I7_R0_LEPEDEA", "I8_R0_LEPEDEA" ];
41792000   Elena.Budnik   new REMOTEDATA
29
   
d4071e27   Elena.Budnik   makeProxy with en...
30
	protected $DDserverXml, $DDserverXmlName;
41792000   Elena.Budnik   new REMOTEDATA
31
	
41792000   Elena.Budnik   new REMOTEDATA
32
33
34
35
36
37
38
39
	
	public function html_encode($param) 
	{
		$pairs = array(" " => "%20"); 
		
		return strtr($param,$pairs); 
	}
	
217c83f2   Elena.Budnik   correct CDAWEB tr...
40
41
42
43
44
45
46
	public function html_decode($param) 
	{
		$pairs = array("%20" => " "); 
		
		return strtr($param,$pairs); 
	}
	
41792000   Elena.Budnik   new REMOTEDATA
47
	public function init() 	
8c0d5a9f   Elena.Budnik   correct location ...
48
	{			
35322ea6   Elena.Budnik   case of absence o...
49
50
		error_log("CDAWEB Proxy creation on ".date("Y-m-d\TH:i:s").PHP_EOL,3,log);
		error_log("CDAWEB Proxy creation on ".date("Y-m-d\TH:i:s").PHP_EOL,3,err);
217c83f2   Elena.Budnik   correct CDAWEB tr...
51
52
53
		
        error_reporting(ALL);
        
d1327dc0   Benjamin Renard   Fix RemoteData ac...
54
		$this->initStreamContext();
05fbcab0   Elena.Budnik   DDServices reorga...
55
		
7216c535   Elena.Budnik   added samplingCDA...
56
57
58
59
60
61
62
63
64
65
		$this->getAllSpaseDatasets();
		
		// predefined samplings
		if (file_exists(RemoteData."/samplingCDAWEB.xml")) {
            $samplingsDom = new DomDocument("1.0");
            if ($samplingsDom->load(RemoteData."/samplingCDAWEB.xml")) {
                $this->xp = new domxpath($samplingsDom);
            }
		}
	} 
41792000   Elena.Budnik   new REMOTEDATA
66
		
d1327dc0   Benjamin Renard   Fix RemoteData ac...
67
68
	private function openConnection() 
	{	
41792000   Elena.Budnik   new REMOTEDATA
69
70
		$this->ch = curl_init();			 
		curl_setopt($this->ch, CURLOPT_RETURNTRANSFER, 1);
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
71
		curl_setopt($this->ch, CURLOPT_TIMEOUT, 600);
d1327dc0   Benjamin Renard   Fix RemoteData ac...
72
73
74
75
76
77
78
79
80
		// Add proxy definition
		$PROXY_HOST=getenv('PROXY_HOST');
		$PROXY_USERPWD=getenv('PROXY_USERPWD');
		if (!empty($PROXY_HOST)) {
			curl_setopt($this->ch, CURLOPT_PROXY, getenv('PROXY_HOST'));
			if (!empty($PROXY_USERPWD)) {
				curl_setopt($this->ch, CURLOPT_PROXYUSERPWD, getenv('PROXY_USERPWD'));
			}
		}
41792000   Elena.Budnik   new REMOTEDATA
81
82
83
84
85
86
	} 
		  
	private function closeConnection()
	{
		curl_close($this->ch);	
	}
d1327dc0   Benjamin Renard   Fix RemoteData ac...
87
88
89

	private function initStreamContext()
	{
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
90
91
92
93
94
		$context = array(
			'http' => array(
				'timeout' => 600,
			),
		);
d1327dc0   Benjamin Renard   Fix RemoteData ac...
95
96
97
		$PROXY_HOST=getenv('PROXY_HOST');
		$PROXY_USERPWD=getenv('PROXY_USERPWD');
		if (!empty($PROXY_HOST)) {
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
98
99
			$context['http']['proxy'] = "tcp://$PROXY_HOST";
			$context['http']['request_fulluri'] = TRUE;
d1327dc0   Benjamin Renard   Fix RemoteData ac...
100
101
102
			if (!empty($PROXY_USERPWD)) {
				$context['http']['header'] = "Proxy-Authorization: Basic ".base64_encode($PROXY_USERPWD);
                        }
d1327dc0   Benjamin Renard   Fix RemoteData ac...
103
		}
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
104
		stream_context_set_default($context);
41792000   Elena.Budnik   new REMOTEDATA
105
	}
d4071e27   Elena.Budnik   makeProxy with en...
106
	
217c83f2   Elena.Budnik   correct CDAWEB tr...
107
108
109
110
	/**
	* Using CDAS REST APIs get ALL CDAWEB dataset descriptions
	* Create an array obsGroupsIds[$obsGroupId][$obsId][$insId][$dsId]
	*/
41792000   Elena.Budnik   new REMOTEDATA
111
	protected function getRemoteTree()	
217c83f2   Elena.Budnik   correct CDAWEB tr...
112
113
	{
        $this->openConnection();
217c83f2   Elena.Budnik   correct CDAWEB tr...
114
        $this->obsGroupsIds = array();   
775592ff   Elena.Budnik   bug in insNodes a...
115

9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
	$dom = NULL;
	$reuse_cache_file = FALSE;
	if (file_exists($this->location."/datasets.xml")) {
		if (time() - filemtime($this->location."/datasets.xml") < 86400) {
			$dom = new DOMDocument();
			if ($dom->load($this->location."/datasets.xml")) {
				echo "Re-use cache file ".$this->location."/datasets.xml".PHP_EOL;
				$reuse_cache_file = TRUE;
			}
		}
	}

	if (!$reuse_cache_file) {
		$dom = $this->loadFromCDAWebWS("datasets", array());
		if (!$dom) {
			error_log('Cannot retrieve CDAWeb datasets list');
			return;
		}
		$dom->save($this->location."/datasets.xml");
	}

        $datasets = $dom->getElementsByTagName("DatasetDescription");
217c83f2   Elena.Budnik   correct CDAWEB tr...
138
139
    
        echo "All CDAWeb datasets : ".$datasets->length.PHP_EOL;
05fbcab0   Elena.Budnik   DDServices reorga...
140

217c83f2   Elena.Budnik   correct CDAWEB tr...
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
        foreach ($datasets as $ds) {
        
            $insType = $ds->getElementsByTagName("InstrumentType")->item(0)->nodeValue;
            
            if (in_array ($this->html_encode($insType), $this->validInstrumentTypes )) {

                $id = $ds->getElementsByTagName("Id")->item(0)->nodeValue;
                            
                $insId = $ds->getElementsByTagName("Instrument")->item(0)->nodeValue;
                if (! in_array ($insId, $this->excludeIns )) {
                    $obsId = "UNKNOWN";
                    if ($ds->getElementsByTagName("Observatory")->length > 0)  
                        $obsId = $ds->getElementsByTagName("Observatory")->item(0)->nodeValue;
                        
                    $obsGroupId = "UNKNOWN"; 
                    if ($ds->getElementsByTagName("ObservatoryGroup")->length > 0)
                        $obsGroupId = $ds->getElementsByTagName("ObservatoryGroup")->item(0)->nodeValue;
                    
                    if ($obsId == "UNKNOWN") {
                         $obsId = $obsGroupId;
                    }  
                    
                    if ( $obsGroupId == "UNKNOWN" ) {
                        $obsGroupId = $obsId;
                    }  
                    
                    if ( $obsGroupId != "UNKNOWN" ) {
                        $this->obsGroupsIds[$obsGroupId][$obsId][$insId][]  = $id;  
                    }
            }
        }
        }
41792000   Elena.Budnik   new REMOTEDATA
173
	}
217c83f2   Elena.Budnik   correct CDAWEB tr...
174
175
176
	 
	protected function createObservatoryNode($id, $grpId) 
	{
155b6b3c   Elena.Budnik   other approach to...
177
		$obsNode = $this->domAmda->createElement('observatory');
41792000   Elena.Budnik   new REMOTEDATA
178
				
155b6b3c   Elena.Budnik   other approach to...
179
180
		$obsNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($grpId).":".$this->param2dd($id));
		$obsNode->setAttribute("name",$id);
41792000   Elena.Budnik   new REMOTEDATA
181
		
155b6b3c   Elena.Budnik   other approach to...
182
		return $obsNode;
41792000   Elena.Budnik   new REMOTEDATA
183
184
	}
	
217c83f2   Elena.Budnik   correct CDAWEB tr...
185
186
	protected function createInstrumentNode($id, $obsId, $groupId) 
	{
41792000   Elena.Budnik   new REMOTEDATA
187
188
189
190
191
192
193
194
195
196
197
		$insNode = $this->domAmda->createElement('instrument');
				
		$insNode->setAttribute("xml:id",$this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsId).":".$id);
		$insNode->setAttribute("name",$id);
	  //	$insNode->setAttribute("description",$ins[1]);
		
		return $insNode;
	}
	
	protected function createDatasetNode($dsId)
	{		
217c83f2   Elena.Budnik   correct CDAWEB tr...
198
199
		if ( !array_key_exists($dsId, $this->CDAWEB ))
			return null; // no description in SpaseRegistry => we do not add this dataset
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
200

41792000   Elena.Budnik   new REMOTEDATA
201
		$dsNode = $this->domAmda->createElement('dataset');
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
202

41792000   Elena.Budnik   new REMOTEDATA
203
		
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
204
205
206
207
		$obj = $this->loadFromCDAWebWS("datasets", array("idPattern" => $dsId), TRUE);
		if (!$obj) {
			return NULL;
		}
41792000   Elena.Budnik   new REMOTEDATA
208
			
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
209
		$dataSet = $obj->DatasetDescription;
41792000   Elena.Budnik   new REMOTEDATA
210
211
212
213
214
215
216
217
218
219
220
221
222
      
		$dsNode->setAttribute("xml:id",$this->baseID.":".$dsId);
		$dsNode->setAttribute("name", $dsId);
		$startTime = $dataSet[0]->TimeInterval->Start;
		$endTime =  $dataSet[0]->TimeInterval->End;
		$label = $dataSet[0]->Label; 
		$url = $dataSet[0]->DatasetLink[0]->Url;
		$notesUrl = $dataSet[0]->Notes; 
		$piAffilation = $dataSet[0]->PiAffiliation;
		$piName = $dataSet[0]->PiName; 
		$sampling = $this->getDatasetSpaseDescription($dsId);
		$this->updateDDServerXml("GlobalStart",$startTime);
		$this->updateDDServerXml("GlobalStop",$endTime);
1b0e0631   Elena.Budnik   unique name for obs
223
		
217c83f2   Elena.Budnik   correct CDAWEB tr...
224
225
226
		// no SPASE description
		if ($sampling == -1) return null;
		
7216c535   Elena.Budnik   added samplingCDA...
227
228
		if ($sampling <= 0)	{
            // check if there is file with predefined samplings
7216c535   Elena.Budnik   added samplingCDA...
229
230
231
232
233
234
235
            if ($this->xp) {
                $set = $this->xp->query("//dataset[@xml:id='".$dsId."']");
                if ($set->length > 0) {
                    $sampling = $set->item(0)->nodeValue;
                    echo "from samplingCDAWEB : ".$dsId." : ".$sampling.PHP_EOL;
                    $this->updateDDServerXml("MinSampling",$sampling);	
                }  else {                    
217c83f2   Elena.Budnik   correct CDAWEB tr...
236
                    echo " No sampling : ".$dsId.PHP_EOL;
7216c535   Elena.Budnik   added samplingCDA...
237
238
239
240
241
242
243
244
                    error_log("! No sampling and No dataset in samplingCDAWEB defined : ".$dsId." : ".$sampling.PHP_EOL,3,err);
                }
            } else {
                echo " No xpath ".PHP_EOL;
                error_log("! No sampling and No xpath defined : ".$dsId." : ".$sampling.PHP_EOL,3,err);
            }
			
		}
873ce7b2   Elena.Budnik   CDAWEB exclude so...
245
        else 
41792000   Elena.Budnik   new REMOTEDATA
246
			$this->updateDDServerXml("MinSampling",$sampling);	
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
247

155b6b3c   Elena.Budnik   other approach to...
248
		$dsNode->setAttribute('spaseUrl',$this->CDAWEB[$dsId]);
41792000   Elena.Budnik   new REMOTEDATA
249
	//   $dsNode->setAttribute('masterCdf',$this->existsMasterCdf($dsId));
155b6b3c   Elena.Budnik   other approach to...
250
		$dsNode->setAttribute("desc", "$label; $startTime - $endTime");
21b939db   Elena.Budnik   dataStart, dataSt...
251
252
		$dsNode->setAttribute("dataStart", $startTime); 
		$dsNode->setAttribute("dataStop", $endTime); 
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
253

41792000   Elena.Budnik   new REMOTEDATA
254
		$parameterNodes = $this->createParameterNodes($dsId);
217c83f2   Elena.Budnik   correct CDAWEB tr...
255
		foreach ($parameterNodes as $parameterNode)
41792000   Elena.Budnik   new REMOTEDATA
256
257
258
259
260
261
262
263
264
		{
			$dsNode->appendChild($parameterNode);
		}
		
		return $dsNode;
	}
	
	protected function createParameterNodes($dsId)
	{	
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
265
266
267
	 	$obj = $this->loadFromCDAWebWS("datasets/".$dsId."/variables", array(), TRUE);
		if (!$obj)
			return array();
41792000   Elena.Budnik   new REMOTEDATA
268
	 	$parameters = $obj->VariableDescription;
d4071e27   Elena.Budnik   makeProxy with en...
269

41792000   Elena.Budnik   new REMOTEDATA
270
		$paramNodes = array();
217c83f2   Elena.Budnik   correct CDAWEB tr...
271

41792000   Elena.Budnik   new REMOTEDATA
272
273
274
275
276
277
278
279
280
281
282
283
284
		foreach ($parameters as $param)
		{
			$paramNode = $this->domAmda->createElement('parameter');
			$paramNode->setAttribute("xml:id", $this->baseID.":".$dsId.":".$param->Name);
			$paramNode->setAttribute("name", $param->Name);
			
			$paramNodes[] = $paramNode;
		}	
		
		return $paramNodes;
	}
	
	protected function getDatasetSpaseDescription($dsID)
217c83f2   Elena.Budnik   correct CDAWEB tr...
285
	{		 
41792000   Elena.Budnik   new REMOTEDATA
286
287
		if ( !array_key_exists($dsID, $this->CDAWEB ))
			return -1; // no description in SpaseRegistry
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
288
289
290
		
		if (!$this->loadSpaseResource($this->CDAWEB[$dsID], $this->spase_res)) return -100;
	
155b6b3c   Elena.Budnik   other approach to...
291
		$messages = $this->spase_res->getElementsByTagName('Message');
41792000   Elena.Budnik   new REMOTEDATA
292
293
294
		
		if ($messages->length > 0)
		{
217c83f2   Elena.Budnik   correct CDAWEB tr...
295
296
    // 		foreach ($messages as $message)
    // 					echo $message->nodeValue.PHP_EOL;
41792000   Elena.Budnik   new REMOTEDATA
297
298
299
			return -2;   // no description in  www-spase
		}
		
217c83f2   Elena.Budnik   correct CDAWEB tr...
300
301
//  		$instrument = $this->spase_res->getElementsByTagName('InstrumentID');
//         echo "  SPASE :   instrument ".$instrument->item(0)->nodeValue.PHP_EOL;
41792000   Elena.Budnik   new REMOTEDATA
302
		
155b6b3c   Elena.Budnik   other approach to...
303
		$cadence = $this->spase_res->getElementsByTagName('Cadence');
41792000   Elena.Budnik   new REMOTEDATA
304
		
155b6b3c   Elena.Budnik   other approach to...
305
		if ($cadence->length == 0)
41792000   Elena.Budnik   new REMOTEDATA
306
307
308
309
310
311
312
313
314
315
			return -3; // no cadence in spase xml
			
		$sampling = $this->cadence2sampling($cadence->item(0)->nodeValue);
		
		return $sampling; 	 
	}
	
	public function cadence2sampling($cadence)
	{
		$scale = array('S' => 1, 'M' => 60, 'H' => '3600');
41792000   Elena.Budnik   new REMOTEDATA
316
		$value = substr($cadence,2,strlen($cadence)-3);
41792000   Elena.Budnik   new REMOTEDATA
317
		$units = substr($cadence,-1); 
0fb01225   Elena.Budnik   float sampling
318
319
		$sampling = floatval($value)*$scale[$units];
		
41792000   Elena.Budnik   new REMOTEDATA
320
321
322
		return $sampling;
	}
	
217c83f2   Elena.Budnik   correct CDAWEB tr...
323
324
325
326
	/**
	* Get IDs and SPASE URLs of all CDAWEB SPASE-defined datasets ( SPASEql )
	* Create an array $CDAWEB[$ID] = SpaseID
	*/
41792000   Elena.Budnik   new REMOTEDATA
327
328
	protected function getAllSpaseDatasets()
	{
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
329
330
		$this->initStreamContext();

41792000   Elena.Budnik   new REMOTEDATA
331
		require_once "simple_html_dom.php";
873ce7b2   Elena.Budnik   CDAWEB exclude so...
332

9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
333
334
335
		if (file_exists($this->location."/NumericalData.html"))
			rename($this->location."/NumericalData.html",$this->location."/NumericalData.html.bak");

41792000   Elena.Budnik   new REMOTEDATA
336
		
d4071e27   Elena.Budnik   makeProxy with en...
337
		// if cannot reach CDAWEB Spase Registry use an old file
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
338
339
		if (!copy(CDAWebConfigClass::$spaseRegistry, $this->location."/NumericalData.html")) {
				error_log('ERROR');
8c0d5a9f   Elena.Budnik   correct location ...
340
				copy($this->location."/NumericalData.html.bak", $this->location."/NumericalData.html");
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
341
342
343
344
345
346
347
348
349
		}

		$doc = new DOMDocument('1.0', 'utf-8');
		if (!$doc->loadHTMLFile($this->location.'/NumericalData.html')) {
			error_log("Cannot load ".$this->location.'/NumericalData.html');
		}

		$xpath = new DOMXPath($doc);
		$ids = $xpath->query("//td[@class='Spase.URL.ProductID']");
d4071e27   Elena.Budnik   makeProxy with en...
350

41792000   Elena.Budnik   new REMOTEDATA
351
		foreach ($ids as $id)
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
352
353
354
355
356
357
		{
			$key = $id->nodeValue;
			$spase = $id->nextSibling->nodeValue;
			if (!empty($key) && (strlen($spase) > 25)) {
				$this->CDAWEB[$key] = $spase;
			}
41792000   Elena.Budnik   new REMOTEDATA
358
		}
217c83f2   Elena.Budnik   correct CDAWEB tr...
359

8c0d5a9f   Elena.Budnik   correct location ...
360
		error_log("SPASE-defined CDAWEB datasets from ".CDAWebConfigClass::$spaseRegistry." : ".count($this->CDAWEB).PHP_EOL,3,log);
05fbcab0   Elena.Budnik   DDServices reorga...
361
		echo "SPASE-defined CDAWEB datasets : ".count($this->CDAWEB).PHP_EOL;
41792000   Elena.Budnik   new REMOTEDATA
362
363
364
365
366
367
368
369
370
371
372
373
374
375
	}
	
	protected function initDDServerXml($ds, $ins, $obs)
	{
		$this->DDserverXml = new DomDocument("1.0");
		$this->DDserverXmlName =  $this->DDserverDir."/".$ds.".xml";
		$rootNode = $this->DDserverXml->createElement('VI');
		$this->DDserverXml->appendChild($rootNode);
		$rootNode->appendChild($this->DDserverXml->createElement("DataBaseID", $this->baseID));
		$rootNode->appendChild($this->DDserverXml->createElement("Mission", $obs));
		$rootNode->appendChild($this->DDserverXml->createElement("Instrument", $ins));
		$rootNode->appendChild($this->DDserverXml->createElement("DataSetID", $ds));
		$rootNode->appendChild($this->DDserverXml->createElement("GlobalStart"));
		$rootNode->appendChild($this->DDserverXml->createElement("GlobalStop"));
1b0e0631   Elena.Budnik   unique name for obs
376
377
		$rootNode->appendChild($this->DDserverXml->createElement("MinSampling"));
		// fill value by default - if different - add into param description
2f2f6cee   Elena.Budnik   FillValue by default
378
		$rootNode->appendChild($this->DDserverXml->createElement("FillValue", -1.0E31));
41792000   Elena.Budnik   new REMOTEDATA
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
	}
	
	protected function updateDDServerXml($target, $value)
	{
		$node = $this->DDserverXml->getElementsByTagName($target);
		$node->item(0)->nodeValue = $value;
	}
	
	protected function saveDDServerXml()
	{
		$this->DDserverXml->save($this->DDserverXmlName);
	}
	
	protected function existsMasterCdf($dsId)
	{
d1327dc0   Benjamin Renard   Fix RemoteData ac...
394
		$this->initStreamContext();
41792000   Elena.Budnik   new REMOTEDATA
395
396
397
398
399
		$file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf";
		$file_headers = @get_headers($file);
		
		if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
			$exists = false;
8c0d5a9f   Elena.Budnik   correct location ...
400
			error_log("INFO:   no master CDF ".$dsId.PHP_EOL,3,log);
41792000   Elena.Budnik   new REMOTEDATA
401
402
403
404
405
406
407
408
409
410
		}
		else {
			$exists = true;
		}
		
		return $exists;
	}
		 
	protected function getMasterCdf($dsId)
	{
d1327dc0   Benjamin Renard   Fix RemoteData ac...
411
		$this->initStreamContext();
41792000   Elena.Budnik   new REMOTEDATA
412
413
414
415
416
417
418
419
420
421
422
		$file = CDAWebConfigClass::$masterUrl.strtolower($dsId)."_00000000_v01.cdf";
		$file_headers = @get_headers($file);
		
		if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
			 return false;
		}		 
		return $file;		 	 
	}
	
	public function getData($ds, $start, $stop)
	{
d1327dc0   Benjamin Renard   Fix RemoteData ac...
423
		$this->initStreamContext();
41792000   Elena.Budnik   new REMOTEDATA
424
		$this->openConnection();
41792000   Elena.Budnik   new REMOTEDATA
425

9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
426
427
428
429
430
431
432
		$res = $this->loadFromCDAWebWS("datasets/$ds/orig_data/$start,$stop/", array());
		if (!$res) {
			return array();
		}

		if ($res->getElementsByTagName("html")->length > 0)
		{
8c0d5a9f   Elena.Budnik   correct location ...
433
434
			error_log("ERROR no response for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
			error_log($res->saveXML(),3,err);
14659123   Elena.Budnik   return empty arra...
435
436
			$this->closeConnection();
			return array();
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
437
		}
41792000   Elena.Budnik   new REMOTEDATA
438
	   
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
439
		$fileNames = $res->getElementsByTagName("Name");
41792000   Elena.Budnik   new REMOTEDATA
440
441
442
443
444
445
446
447
448
449
450
451
	   	   
		$nc_prefix = strlen($ds) > RemoteDataCenterClass::$MAX_VI_NAME_LENGTH ? 
				substr(strtolower($ds),0,RemoteDataCenterClass::$MAX_VI_NAME_LENGTH - 1): strtolower($ds);
		$files = array();
	   
		for ($i = 0; $i < $fileNames->length;  $i++)
	   {
			$fileName = $fileNames->item($i);
			$url = $fileName->nodeValue;
			$file_headers = @get_headers($url);
		
			if(!$file_headers || $file_headers[0] == 'HTTP/1.1 404 Not Found') {
8c0d5a9f   Elena.Budnik   correct location ...
452
				error_log("ERROR  404 Not Found for : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
41792000   Elena.Budnik   new REMOTEDATA
453
454
455
456
				continue;
			}	
			$temp = explode('/',$url);
			$destination = $temp[count($temp)-1];
7216c535   Elena.Budnik   added samplingCDA...
457

41792000   Elena.Budnik   new REMOTEDATA
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
			if (copy($url, $destination))
			{
				$ncFile = $this->convert2nc($destination);
				if (strlen($ncFile) > RemoteDataCenterClass::$MAX_NAME_LENGTH)
				{
					$ncFileTrue = $nc_prefix."_".time().$i.".nc";
					rename($ncFile, $ncFileTrue);
					$files[] = $ncFileTrue;
				}
				else
				{
					$files[] = $ncFile;
				}
			}
			else
			{
8c0d5a9f   Elena.Budnik   correct location ...
474
				error_log("ERROR cannot copy files : ".$ds." : ".$start." - ".$stop.PHP_EOL,3,err);
41792000   Elena.Budnik   new REMOTEDATA
475
476
477
478
479
480
481
482
483
484
			}
	   }
	   
		$this->closeConnection();
		
		return $files;
	}
	
	protected function convert2nc($file)
	{
0fb01225   Elena.Budnik   float sampling
485
		system("cdfnew2nc $file");		 
41792000   Elena.Budnik   new REMOTEDATA
486
487
		$ncFile = str_replace(".cdf", ".nc", $file);
		
8c0d5a9f   Elena.Budnik   correct location ...
488
489
490
491
 		if (!file_exists($ncFile)) {
			error_log("ERROR while converting ".$file.PHP_EOL,3,err);
			return false; 
 		}
41792000   Elena.Budnik   new REMOTEDATA
492
493
494
495
496
		unlink($file);	
		
		return $ncFile;
	}
	
0ec21281   Elena.Budnik   reorganization + ...
497
498
	public function getDatasetInfo($ds)
	{
35322ea6   Elena.Budnik   case of absence o...
499
500
501
		if (!$this->existsMasterCdf($ds)) return false;
		
		$masterCdf = $this->getMasterCdf($ds);
0ec21281   Elena.Budnik   reorganization + ...
502
		$localCdf = strtolower($ds).".cdf";
35322ea6   Elena.Budnik   case of absence o...
503
		if (!copy($masterCdf, $localCdf)) return false;
0ec21281   Elena.Budnik   reorganization + ...
504
505
506
507
508
	 
		$infoFile = $this->convert2nc($localCdf);
	  
		return $infoFile;
	}
155b6b3c   Elena.Budnik   other approach to...
509
510
	
	public function makeProxy()
217c83f2   Elena.Budnik   correct CDAWEB tr...
511
	{ 
155b6b3c   Elena.Budnik   other approach to...
512
513
514
515
516
517
518
519
520
521
		$this->spase_res = new DomDocument("1.0");
		$this->domAmda = new DOMDocument('1.0', 'utf-8');
		$this->domAmda->formatOutput = TRUE;
		$this->domAmda->preserveWhiteSpace = FALSE;
		
		$dataRoot = $this->domAmda->createElement('dataRoot');
		$dataRoot->setAttribute('xml:id', 'myRemoteData-treeRootNode');
		$this->domAmda->appendChild($dataRoot);	
		
		$this->dataCenter=$this->domAmda->createElement('dataCenter');
8c0d5a9f   Elena.Budnik   correct location ...
522
523
	 	$this->setDataCenterAttributes();
	 	
155b6b3c   Elena.Budnik   other approach to...
524
		$dataRoot->appendChild($this->dataCenter);
217c83f2   Elena.Budnik   correct CDAWEB tr...
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550

        foreach ($this->obsGroupsIds as $groupId => $obss)
        {
            if (in_array($groupId, $this->excludeGroup)) continue;
            echo $groupId.PHP_EOL; 
            $obsNodes = array();
            foreach ($obss as $obs => $inss)
            {
                // ! observatory OMNI is taken for observatoriesGroup OMNI only !
                if (substr($obs,0,4) == "OMNI" && substr($groupId,0,4) != "OMNI"){
                    echo 'OMNI      '.$obs.' '.$missionId.PHP_EOL;
                    continue;
                } 
                $insNodes = array();
                // Create instrument nodes
                foreach ($inss as $ins => $dss)
                {
                    $dsNodes = array();
                    // Create datasets nodes
                    foreach ($dss as $ds)
                    {	
                        $dsId = $this->baseID.":".$ds;
                            
                        if (!$this->domAmda->getElementById($dsId)){
                            $this->initDDServerXml($ds,$ins,$obs);
                            $dsNode = $this->createDatasetNode($ds);
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
551

217c83f2   Elena.Budnik   correct CDAWEB tr...
552
553
554
555
556
557
558
559
560
561
562
                            if ($dsNode){
                                $this->saveDDServerXml();
                                $dsNodes[] = $dsNode;
                            }
                        }	
                    } // foreach ($dss as $ds)
                    
                    if (!empty($dsNodes))
                    {
                            // last $spase_res : instrument should be the same
                            $insSpaseId = $this->getInstrumentSpase();   
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
563

217c83f2   Elena.Budnik   correct CDAWEB tr...
564
565
                            if (!$insSpaseId || is_array($insSpaseId)) 
                                $insSpaseId = $ins;
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
566

217c83f2   Elena.Budnik   correct CDAWEB tr...
567
568
569
                            $obsSpaseId = strtolower($this->getObservatorySpase());
                            if (!$obsSpaseId ) 
                                $obsSpaseId = strtolower($obs);
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
570

217c83f2   Elena.Budnik   correct CDAWEB tr...
571
                            $insId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId).":".$this->param2dd($insSpaseId);
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
572

217c83f2   Elena.Budnik   correct CDAWEB tr...
573
574
575
576
577
578
579
580
581
582
583
                            if (!($insNode = $this->domAmda->getElementById($insId))){
                                $insNode = $this->createInstrumentNode($insSpaseId, $obsSpaseId, $groupId);
                            }
                            
                            foreach ($dsNodes as $dsNode){
                                $insNode->appendChild($dsNode);
                            }
                                
                            $insNodes[] = $insNode;
                        }
                    } // foreach ($inss as $ins => $dss)
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
584

217c83f2   Elena.Budnik   correct CDAWEB tr...
585
586
587
588
589
590
591
592
593
594
595
596
597
598
                    if (!empty($insNodes))
                    {					
                        $obsId = $this->baseID.":".$this->param2dd($groupId).":".$this->param2dd($obsSpaseId);
                        
                        if (!($obsNode = $this->domAmda->getElementById($obsId))){   
                            $obsNode = $this->createObservatoryNode($obsSpaseId, $groupId);
                        }
                        
                        foreach ($insNodes as $insNode){
                            $obsNode->appendChild($insNode);
                        }
                        $obsNodes[] = $obsNode;
                    }
                } // foreach ($obss as $obs => $inss)
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
599

217c83f2   Elena.Budnik   correct CDAWEB tr...
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
                
                if (!empty($obsNodes))
                {
                    $missionId = $this->baseID.":".$this->param2dd($groupId);
                    
                    if (!($missionNode = $this->domAmda->getElementById($missionId))){
                        $missionNode = $this->createMissionNode($groupId);
                    }
                        
                    foreach ($obsNodes as $obsNode)
                    {
                        // observatory == mission
                        if (strtolower($obsNode->getAttribute('name')) == strtolower($missionNode->getAttribute('name'))) {
                            $insNodes = $obsNode->getElementsByTagName("instrument");
                            foreach ($insNodes as $insNode) {
                                $missionNode->appendChild($insNode->cloneNode(true));
                            }
                        }
                        else 
                            $missionNode->appendChild($obsNode);
                    }
                    
                    $this->dataCenter->appendChild($missionNode); 
                }                                
            } 

05fbcab0   Elena.Budnik   DDServices reorga...
626
		$this->closeConnection();
155b6b3c   Elena.Budnik   other approach to...
627
628
	}
	
8c0d5a9f   Elena.Budnik   correct location ...
629
630
	protected function createMissionNode($groupId)
	{
155b6b3c   Elena.Budnik   other approach to...
631
632
633
634
635
636
637
638
		$missionId = $this->baseID.":".$this->param2dd($groupId);
		$missionNode = $this->domAmda->createElement('mission');
		$missionNode->setAttribute("xml:id",$missionId);
		$missionNode->setAttribute("name",$groupId);
		
		return $missionNode;
	}
	
8c0d5a9f   Elena.Budnik   correct location ...
639
640
	protected function getInstrumentSpase()
	{	
62372d94   Elena.Budnik   RemoveVi
641
		$this->insXML = new DomDocument("1.0");
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
642

155b6b3c   Elena.Budnik   other approach to...
643
644
		if ($this->spase_res){
			$instrument = $this->spase_res->getElementsByTagName('InstrumentID');
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
645

155b6b3c   Elena.Budnik   other approach to...
646
			if ($instrument->length > 0) {
217c83f2   Elena.Budnik   correct CDAWEB tr...
647
                if ($instrument->length == 1) {
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
648
649
		    $this->loadSpaseResource($instrument->item(0)->nodeValue, $this->insXML);

217c83f2   Elena.Budnik   correct CDAWEB tr...
650
                    return $this->getIdFromSpase($instrument->item(0)->nodeValue);
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
651
652
		}
		else {
217c83f2   Elena.Budnik   correct CDAWEB tr...
653
654
655
656
657
658
659
660
661
                    // $insIds = "";
                    $insIds = array();
                    for  ($i = 0; $i < $instrument->length; $i++ ) {                                            
//                         $insIds  .= $this->getIdFromSpase($instrument->item($i)->nodeValue);
//                         if ($i < $instrument->length - 1) 
//                             $insIds .= "/";
                        $insIds[] = $this->getIdFromSpase($instrument->item($i)->nodeValue);
                    }
                    
9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
662
		    $this->loadSpaseResource($instrument->item($instrument->length - 1)->nodeValue, $this->insXML);
217c83f2   Elena.Budnik   correct CDAWEB tr...
663
664
                    return $insIds;
				}
155b6b3c   Elena.Budnik   other approach to...
665
666
667
668
669
670
671
672
				//InstrumentType
				//spase://SMWG/Observatory/ACE
			} 
		}	
		
		return null;
	}
	
8c0d5a9f   Elena.Budnik   correct location ...
673
674
	protected function getObservatorySpase()
	{
62372d94   Elena.Budnik   RemoveVi
675
676
677
678
679
680
681
682
683
684
685
		if ($this->insXML){
			$observatory = $this->insXML->getElementsByTagName('ObservatoryID');
			
			if ($observatory->length > 0) {
				return $this->getIdFromSpase($observatory->item(0)->nodeValue); 			
			} 
		}	
		
		return null;
	}
	
8c0d5a9f   Elena.Budnik   correct location ...
686
687
688
689
690
691
	protected function setDataCenterAttributes(){	
		$this->dataCenter->setAttribute('xml:id', $this->baseID);
	//	$this->dataCenter->setIdAttribute('xml:id', true);
		$this->dataCenter->setAttribute('name', $this->baseID);		
	}
	
41792000   Elena.Budnik   new REMOTEDATA
692
	protected function makeArgumentsList(){}
05fbcab0   Elena.Budnik   DDServices reorga...
693

9f59dba7   Benjamin Renard   Fix CDAWeb tree g...
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761

	private function loadSpaseResource($resourceID, &$dom) {
		if (empty($resourceID)) {
			return FALSE;
		}
		$url = str_replace('spase://', 'https://hpde.io/', $resourceID) . '.xml';
		curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/xml"));
		curl_setopt($this->ch, CURLOPT_URL, $url);
                if (!$dom->loadXML(curl_exec($this->ch))) {
			error_log('Cannot load Spase Resource: '.$resourceID);
			return FALSE;
		}
		return TRUE;
	}

	private function loadFromCDAWebWS($api, $params, $asJSON = FALSE) {
		if (!$this->dataViewURL) {
			curl_setopt($this->ch, CURLOPT_URL, CDAWebConfigClass::$restUrl."/dataviews");
			curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/xml"));

			$res = new DomDocument();
			if (!$res->loadXML(curl_exec($this->ch))) {
				return FALSE;
			}

			if ($res->getElementsByTagName('EndpointAddress')->length == 0) {
				error_log("Problem connect to ".CDAWebConfigClass::$restUrl."/dataviews", 3, err);
				return FALSE;
			}

			$this->dataViewURL = $res->getElementsByTagName('EndpointAddress')->item(0)->nodeValue;
		}
		$url = $this->dataViewURL . '/' . $api;
		if (!empty($params)) {
			$params_str = "";
			foreach ($params as $key => $value) {
				if (!empty($params_str)) {
					$params_str .= "&";
				}
				$params_str .= $key."=".urlencode($value);
			}
			$url .= '?'.$params_str;
		}

		if ($asJSON) {
			curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/json"));
		}
		else {
			curl_setopt($this->ch, CURLOPT_HTTPHEADER, array("Accept: application/xml"));
		}
		curl_setopt($this->ch, CURLOPT_URL, $url);

		$res = curl_exec($this->ch);
		if (!$res) {
			return FALSE;
		}

		if ($asJSON) {
			return json_decode($res);
		}

		$dom = new DOMDocument();
     		if (!$dom->loadXML(curl_exec($this->ch))) {
			return FALSE;
		}

		return $dom;
	}
41792000   Elena.Budnik   new REMOTEDATA
762
763
} 
?>