CDPP / DDServer

Blame view

src/DATA/MANAGER/DDBaseMgr.php 12.1 KB
Edit Raw Normal View History
34cd00b5   Elena.Budnik   new DATAMANAGER 
1
2
3
4
5
6
7
8
9
10
 
<?php

/**
*  @class DDBaseMgr
*  @brief  
*  @arg    
*/
class DDBaseMgr 
{
c8f4ea1d   Elena.Budnik   new cdf2nc, updat... 
11
 
	public $brief, $times, $info, $info_xml, $cache;
34cd00b5   Elena.Budnik   new DATAMANAGER 
12
13
14
 
	private $ViId, $remoteViId;
	private $DDsysDoc, $DDsys;
	private $base, $mission, $instrument;
2f2f6cee   Elena.Budnik   FillValue by default 
15
 
	private $min_sampling, $max_sampling, $fillValue = null;
c8f4ea1d   Elena.Budnik   new cdf2nc, updat... 
16
 
	public $location, $ViDir;
34cd00b5   Elena.Budnik   new DATAMANAGER 
17
18
19
20
 
	public $globalStart = null, $globalStop = null;
	
   function __construct()
	{
2c71cf83   Elena.Budnik   THEMIS update sta... 
21
 
        $this->DDsys = getenv("DDBASE")."/DDsys.xml";
34cd00b5   Elena.Budnik   new DATAMANAGER 
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 
		$this->DDsysDoc = new DomDocument();
		$this->DDsysDoc->preserveWhiteSpace = false;
		$this->DDsysDoc->formatOutput = true;
		$this->DDsysDoc->load($this->DDsys); 	
	}	
	
	public function setViInfo($brief)
	{
		$this->brief = $brief;
		$this->times = $brief."_times.nc";
		$this->info = $brief."_info.nc";
		$this->info_xml = $brief."_info.xml";
		$this->cache = $brief."_cache.nc";
	}
	
	public function setViDir($ViDir)
	{
		$this->ViDir = $ViDir;
	}
7d7445ea   Elena.Budnik   THEMIS remote center 
42
43
44
45
46
 
	public function setRemoteViId($ViId)
	{
		$this->remoteViId = $ViId;
	}
34cd00b5   Elena.Budnik   new DATAMANAGER 
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
 
	public function setViId($ViId)
	{
		$this->ViId = $ViId;
	}
	
	public function setViParentsInfo($base, $mission, $instrument)
	{
		$this->base = $base;
		$this->mission = $mission;
		$this->instrument = $instrument;
	}
	
	public function setViSampling($min_sampling, $max_sampling)
	{
		$this->min_sampling = $min_sampling;
		$this->max_sampling = $max_sampling;		
	}
	
	public function setViLocation($mission_dir, $location)
	{
		$this->location = getenv("DDBASE")."/".$mission_dir."/".$location;	
		$this->ViDir = $this->location;
	}
	
	// For Remote Data Centers
	public function setViInfoFromFile($base, $ds)
	{
		$infoXml = new DomDocument("1.0");
cf197d4f   Benjamin Renard   Fix bug when DDBA... 
75
76
77
78
79
 
		$DDBASE=getenv("DDBASE");
		if (is_link($DDBASE)) {
			$DDBASE=readlink($DDBASE);
		}
		$infoXmlFile = $DDBASE."/../INFO/DDServer/$base/$ds.xml";
34cd00b5   Elena.Budnik   new DATAMANAGER 
80
81
82
83
84
85
 
		
		if (!file_exists($infoXmlFile))
				return -100;
		
		if (!$infoXml->load($infoXmlFile))
				return -10;
2f2f6cee   Elena.Budnik   FillValue by default 
86
34cd00b5   Elena.Budnik   new DATAMANAGER 
87
88
89
90
91
92
93
 
		$this->base = $base;	
		$this->mission = $infoXml->getElementsByTagName("Mission")->item(0)->nodeValue;
		$this->instrument = $infoXml->getElementsByTagName("Instrument")->item(0)->nodeValue;
		$this->min_sampling = $infoXml->getElementsByTagName("MinSampling")->item(0)->nodeValue;
		$this->remoteViId = $ds;
		$this->globalStart = $infoXml->getElementsByTagName("GlobalStart")->item(0)->nodeValue;
		$this->globalStop = $infoXml->getElementsByTagName("GlobalStop")->item(0)->nodeValue;
2f2f6cee   Elena.Budnik   FillValue by default 
94
95
96
 
		$fillValue = $infoXml->getElementsByTagName("FillValue");
		if ($fillValue->length > 0) $this->fillValue = $fillValue->item(0)->nodeValue;
cf197d4f   Benjamin Renard   Fix bug when DDBA... 
97
 
		$this->location = $DDBASE."/".$this->base."/".strtoupper($this->ViId)."/";
34cd00b5   Elena.Budnik   new DATAMANAGER 
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
 
		$this->ViDir = $this->location;
		//--------- Check if additional Info should be added to VI
// 		$theFormat = $infoXml->getElementsByTagName("Format")->item(0);
// 		if ($theFormat != NULL)
// 			$AddInfo = ($theFormat->nodeValue == "Text");
// 		if (!$AddInfo) 
// 			$AddInfo = ($infoXml->getElementsByTagName("AdditionalInfoNeed")->item(0) != NULL);
		
		return 0;
	}
	
	public function viExists($ViId, $baseId)
	{
		$XPath = new DOMXpath($this->DDsysDoc);
		
		$dataSet = $XPath->query("//NAME[.='$ViId']");
		
		if ($dataSet->item(0) != NULL)
		{						
			if ($dataSet->item(0)->getAttribute("base") != $baseId) return false;
			else return true;			
		}
		
		return false;	
	}
	
	public function createVi()
	{
		 $this->makeDDsys();
		 $this->makeAllInLocation();		
	}
62372d94   Elena.Budnik   RemoveVi 
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
 
	public function deleteVi($ViId, $baseId)
	{
		$XPath = new DOMXpath($this->DDsysDoc);
		
		$dataSet = $XPath->query("//NAME[.='$ViId']");
		
		if ($dataSet->item(0)->getAttribute("base") != $baseId)
			return;
		
		$ViToDelete = $dataSet->item(0)->parentNode;	
		$location = $ViToDelete->getElementsByTagName('LOCATION')->item(0)->nodeValue;
		
		$ViToDelete->parentNode->removeChild($ViToDelete);
         
		$this->DDsysDoc->save($this->DDsys);
		system("makeDDsys");
		
		foreach (glob($location."/*") as $file) unlink($file);
		rmdir($location);          
	}
34cd00b5   Elena.Budnik   new DATAMANAGER 
151
152
153
154
155
156
157
158
159
160
161
162
 
	private function makeAllInLocation()
	{
		if (!is_dir($this->location)) 
			mkdir($this->location, 0775, true);
      $currDir = getcwd();
      
		chdir($this->location);
		if (!file_exists($this->times)) system("TimesUpdate -r ".$this->times." ".$this->brief."_[0-9]*.nc");
		if (!file_exists($this->info_xml)) $this->makeInfoXml();
		if (!file_exists($this->info)) system("infoLocal2nc ".$this->info_xml." ".$this->info);
		if (!file_exists("clean")) 
		{
5f5e337d   Benjamin Renard   First step to rem... 
163
164
 
			copy(getenv("DDBASEBIN")."/Cache.template", $this->cache);         
			$SED = "sed 's/NAME/".$this->brief."/g' ".getenv("DDBASEBIN")."/Clean.template > clean";
34cd00b5   Elena.Budnik   new DATAMANAGER 
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
 
			exec($SED);
			chmod("clean", 0755);
		} 
		
		chdir($currDir);
	}
	
	private function makeInfoXml() 
	{
		$xml_dom = new DomDocument("1.0");
		$rootElement = $xml_dom->createElement("VI");
		if ($this->base != "LOCAL") 
			$rootElement->appendChild($xml_dom->createElement("DataBaseID",$this->base));
		$rootElement->appendChild($xml_dom->createElement("Mission",$this->mission));
		$rootElement->appendChild($xml_dom->createElement("Instrument",$this->instrument));
		if ($this->base != "LOCAL") 
			$rootElement->appendChild($xml_dom->createElement("DataSetID",$this->remoteViId));
		$rootElement->appendChild($xml_dom->createElement("GlobalStart", $this->globalStart));
		$rootElement->appendChild($xml_dom->createElement("GlobalStop", $this->globalStop));
		$rootElement->appendChild($xml_dom->createElement("MinSampling", $this->min_sampling));
		if ($this->max_sampling > $this->min_sampling)  
			$rootElement->appendChild($xml_dom->createElement("MaxSampling", $this->max_sampling));
2f2f6cee   Elena.Budnik   FillValue by default 
187
188
 
		if ($this->fillValue)  
			$rootElement->appendChild($xml_dom->createElement("FillValue", $this->fillValue));
34cd00b5   Elena.Budnik   new DATAMANAGER 
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
 
		$rootElement->appendChild($xml_dom->createElement("LocalStart"));
		$rootElement->appendChild($xml_dom->createElement("LocalStop"));
		$xml_dom->appendChild($rootElement);
		$xml_dom->save($this->location."/".$this->info_xml);		
	}

	private function makeDDsys()
	{
		$newNode = $this->DDsysDoc->createElement("VI");
		$name = $newNode->appendChild($this->DDsysDoc->createElement("NAME",$this->ViId));
		$name->setAttribute('base', $this->base);
		$name->setAttribute('mission', $this->mission);
		$name->setAttribute('instrument', $this->instrument);

		if (substr($this->location, -1) !== "/") 
			$newNode->appendChild($this->DDsysDoc->createElement("LOCATION",$this->location."/")); 
		else 
			$newNode->appendChild($this->DDsysDoc->createElement("LOCATION",$this->location));

		$newNode->appendChild($this->DDsysDoc->createElement("TIMES", $this->times));
		$newNode->appendChild($this->DDsysDoc->createElement("INFO", $this->info));
		$newNode->appendChild($this->DDsysDoc->createElement("CACHE", $this->cache));

		$this->DDsysDoc->documentElement->appendChild($newNode);
			
		$this->DDsysDoc->save($this->DDsys);
		system("makeDDsys");  	
	}
62372d94   Elena.Budnik   RemoveVi 
217
34cd00b5   Elena.Budnik   new DATAMANAGER 
218
219
220
221
222
223
224
225
226
 
// 	public function getRemoteLocation($base, $remoteVi)
// 	{
// 		$XPath = new DOMXpath($this->DDsysDoc);
// 		
// 		$dataSet = $XPath->query("//NAME[.='$ViId']");
// 		$allVis = $this->DDsysDoc-	
// 	
//	}
162f5e88   Elena.Budnik   immediate VI lock 
227
 
	public function lockVi()
34cd00b5   Elena.Budnik   new DATAMANAGER 
228
 
	{
c8f4ea1d   Elena.Budnik   new cdf2nc, updat... 
229
 
		//  Stamp -> this directory is being updated
34cd00b5   Elena.Budnik   new DATAMANAGER 
230
231
232
233
234
 
		touch($this->ViDir."/LOCK");
  
		//	fprintf($STDERR,$ViDir." is LOCKED\n");
	}
162f5e88   Elena.Budnik   immediate VI lock 
235
 
	public function unlockVi()
34cd00b5   Elena.Budnik   new DATAMANAGER 
236
237
238
239
240
 
	{
		if (file_exists($this->ViDir."/LOCK")) unlink($this->ViDir."/LOCK");  
		//	fprintf($STDERR,$ViDir." is UNLOCKED\n");
	}
7d7445ea   Elena.Budnik   THEMIS remote center 
241
 
	public function addRemoteData($id, $ncFiles, $start, $stop, $notFirst = true)
34cd00b5   Elena.Budnik   new DATAMANAGER 
242
 
	{
34cd00b5   Elena.Budnik   new DATAMANAGER 
243
244
245
 
		$WORKING_DIR = getcwd();
		chdir($this->ViDir);
		system("./clean");
2c71cf83   Elena.Budnik   THEMIS update sta... 
246
34cd00b5   Elena.Budnik   new DATAMANAGER 
247
248
249
 
		foreach ($ncFiles as $ncFile)
		{
			//TODO errors
f8a71629   Elena.Budnik   cut long ViId 
250
251
252
 
			//if (!file_exists($ncFile))
			rename($WORKING_DIR."/".$ncFile,$this->ViDir."/".$ncFile);
			system("TimesUpdate -u ".$this->times." ".$ncFile);
34cd00b5   Elena.Budnik   new DATAMANAGER 
253
254
 
		}
7d7445ea   Elena.Budnik   THEMIS remote center 
255
256
257
 
		if ($notFirst)
				system("TimesUpdateNoData ".$this->times." ".$start." ".$stop);
34cd00b5   Elena.Budnik   new DATAMANAGER 
258
 
		chdir($WORKING_DIR);
34cd00b5   Elena.Budnik   new DATAMANAGER 
259
 
	}
0ec21281   Elena.Budnik   reorganization + ... 
260
261
262
 
	
	public function setInfo($infoFile)
	{
cf197d4f   Benjamin Renard   Fix bug when DDBA... 
263
264
265
266
267
 
		$DDBASE=getenv("DDBASE");
		if (is_link($DDBASE)) {
			$DDBASE=readlink($DDBASE);
		}
		$fullInfoName = $DDBASE."/../INFO/bases/".$this->base."/$infoFile";
0ec21281   Elena.Budnik   reorganization + ... 
268
269
270
271
272
 
	 	 
		if (file_exists($fullInfoName)) unlink($fullInfoName);
		//TODO errors		
			//if (!file_exists($infoFile))	
		$WORKING_DIR = getcwd();
0ec21281   Elena.Budnik   reorganization + ... 
273
 
		rename("$WORKING_DIR/$infoFile", $fullInfoName);
0ec21281   Elena.Budnik   reorganization + ... 
274
 
	}
c8f4ea1d   Elena.Budnik   new cdf2nc, updat... 
275
276
277
 
	
	public function updateRemoteStart()
	{
1faf036d   Elena.Budnik   bug in UpdateRemo... 
278
 
		$startStamp = shell_exec("GetStartTime ".$this->location.$this->times);
d63eef4f   Elena.Budnik   typo 
279
 
		$startIso = date("Y-m-d\TH:i:s", intval($startStamp)).substr(number_format(fmod(floatval($startStamp),1),3, '.', ''),1,4)."Z";
c8f4ea1d   Elena.Budnik   new cdf2nc, updat... 
280
281
282
283
284
285
286
287
288
289
290
291
292
 
		
		$xml_dom = new DomDocument("1.0");
		$xml_dom->load($this->location."/".$this->info_xml);
		$startNode = $xml_dom->getElementsByTagName("GlobalStart")->item(0);
		$startNode->nodeValue = $startIso;	
		
		$xml_dom->save($this->location."/".$this->info_xml);
		
		$currDir = getcwd();		
		chdir($this->location);		
		system("infoLocal2nc ".$this->info_xml." ".$this->info);
		chdir($currDir);
	}
1bd736dd   Elena.Budnik   add cleanRemote 
293
d63eef4f   Elena.Budnik   typo 
294
 
	public function updateViStop($globalStop)
2c71cf83   Elena.Budnik   THEMIS update sta... 
295
296
297
 
	{
		$xml_dom = new DomDocument("1.0");
		$xml_dom->load($this->location."/".$this->info_xml);
4a24f977   Elena.Budnik   typo 
298
2c71cf83   Elena.Budnik   THEMIS update sta... 
299
 
		$stopNode = $xml_dom->getElementsByTagName("GlobalStop")->item(0);
4a24f977   Elena.Budnik   typo 
300
 
		$stopNode->nodeValue = $globalStop;
d63eef4f   Elena.Budnik   typo 
301
 
        $xml_dom->save($this->location."/".$this->info_xml);
2c71cf83   Elena.Budnik   THEMIS update sta... 
302
303
304
 
		
		$currDir = getcwd();		
		chdir($this->location);
4a24f977   Elena.Budnik   typo 
305
2c71cf83   Elena.Budnik   THEMIS update sta... 
306
307
308
309
310
 
		system("infoLocal2nc ".$this->info_xml." ".$this->info);
		chdir($currDir);
	}
1bd736dd   Elena.Budnik   add cleanRemote 
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
 
	public function cleanRemote($opt = NULL)
	{
		$res_before = disk_free_space(getenv("DDBASE"));
		$Days = 10;  // default
		$SecDay = 3600*24;
		$nodata = false;

		if (isset($opt)) {
			if ($opt == "NODATA") 
					$nodata = true;
			else {
				$Days = is_numeric($opt) ? $opt : 10;
				settype($Days, "int");
				echo "!!!!!!!!!!!!!!!! Days   ".$Days.PHP_EOL;
			}
		}
		
		$GlobalDelta = $Days*$SecDay;
	
		$xp = new domxpath($this->DDsysDoc);
		$dataSets = $xp->query("//NAME[attribute::base!='LOCAL']");
		
		for ($i = 0; $i < $dataSets->length; $i++) 
		{ 
			$parent = $dataSets->item($i)->parentNode;
			$location = $parent->getElementsByTagName("LOCATION")->item(0)->nodeValue;
 
			if (!is_dir($location)) {
				$parent->parentNode->removeChild($parent);
			}
			else {
				$time_info = $parent->getElementsByTagName("TIMES")->item(0)->nodeValue;
				chdir($location);
				system("./clean");
				touch($location."/LOCK");

				fprintf(STDERR,"In the ".$location."\n");

				/*  Delete No Data Intervals */        
				if ($nodata) {
					system("CleanNoData ".$time_info); 
				}
				/*  Delete Old Files */  
				else {
21ee78c5   Elena.Budnik   always keeps firs... 
355
 
                    $isFirst = true;
1bd736dd   Elena.Budnik   add cleanRemote 
356
 
					foreach (glob("*.gz") as $filename) {
21ee78c5   Elena.Budnik   always keeps firs... 
357
358
359
360
 
                        if ($isFirst) { // we should always keep first file
                            touch($filename);
                            $isFirst = false;
                        }
1bd736dd   Elena.Budnik   add cleanRemote 
361
362
 
						$info = stat($filename);
						if ((time() - $info['atime']) > $GlobalDelta) {
21ee78c5   Elena.Budnik   always keeps firs... 
363
 
						//	fprintf(STDERR, $filename."\n");
1bd736dd   Elena.Budnik   add cleanRemote 
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
 
							system("TimesUpdate -d ".$time_info." ".$filename);
						}
						//fprintf(STDERR, date("Y-m-d",$info['atime'])." - ".date("Y-m-d",$info['mtime'])."\n");
					}
				}

			system("DDClearTimes ".$time_info);
			unlink($location."/LOCK");  
        }
		}
    /* 
		$res_after = disk_free_space(getenv("DDBASE"));
		$difference = ($res_after - $res_before)/1024.0;
		$res_after /=  1024.0;
		fprintf(STDERR,"+ Free Space  ".$difference." kB\n");
		fprintf(STDERR,"  Free Space  ".$res_after." kB\n");*/
		
		$this->DDsysDoc->save($this->DDsys);
		system("makeDDsys");	
	}
88a88fcc   Benjamin Renard   Add script to con... 
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
 

	function convertVIDDTimeToDouble($ViId) {
		$XPath = new DOMXpath($this->DDsysDoc);

		$dataSet = $XPath->query("//NAME[.='$ViId']");

		if ($dataSet->length == 0) {
			echo "[ERROR] NO $ViId in DDsys.xml".PHP_EOL;
			return FALSE;
		}

		$dataSet = $dataSet->item(0);

		$locationNodes = $dataSet->parentNode->getElementsByTagName('LOCATION');

		if ($locationNodes->length == 0) {
			echo "[ERROR] Cannot retrieve LOCATION for $ViId in DDsys.xml".PHP_EOL;
			return FALSE;
		}

		$location = $locationNodes->item(0)->nodeValue;


		if (file_exists($location."/LOCK")) {
			echo "[ERROR] VI $ViId already locked by another process".PHP_EOL;
			return FALSE;
		}

		echo "[INFO] Lock VI $ViId".PHP_EOL;
		touch($location."/LOCK");

		$WORKING_DIR = getcwd();
		chdir($location);

		echo "[INFO] Cleanup VI $ViId".PHP_EOL;
		system("./clean");
		echo "[INFO] Convert data files".PHP_EOL;
		system("ConvertDDTimeToDouble *.nc.gz");

		unlink($location."/LOCK");
		
	}
34cd00b5   Elena.Budnik   new DATAMANAGER 
426
427
 
} 
?>
5f5e337d   Benjamin Renard   First step to rem... 
428