CDPP / DDServer

Blame view

src/DATA/MANAGER/DDBaseMgr.php 12.5 KB
Edit Raw Normal View History
34cd00b5   Elena.Budnik   new DATAMANAGER 
1
2
3
4
5
6
7
8
9
10
 
<?php

/**
*  @class DDBaseMgr
*  @brief  
*  @arg    
*/
class DDBaseMgr 
{
c8f4ea1d   Elena.Budnik   new cdf2nc, updat... 
11
 
	public $brief, $times, $info, $info_xml, $cache;
34cd00b5   Elena.Budnik   new DATAMANAGER 
12
13
14
 
	private $ViId, $remoteViId;
	private $DDsysDoc, $DDsys;
	private $base, $mission, $instrument;
023f1dcc   Benjamin Renard   Add NoCompression... 
15
 
	private $min_sampling, $max_sampling, $fillValue = null, $noCompression;
c8f4ea1d   Elena.Budnik   new cdf2nc, updat... 
16
 
	public $location, $ViDir;
34cd00b5   Elena.Budnik   new DATAMANAGER 
17
18
19
20
 
	public $globalStart = null, $globalStop = null;
	
   function __construct()
	{
2c71cf83   Elena.Budnik   THEMIS update sta... 
21
 
        $this->DDsys = getenv("DDBASE")."/DDsys.xml";
34cd00b5   Elena.Budnik   new DATAMANAGER 
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
 
		$this->DDsysDoc = new DomDocument();
		$this->DDsysDoc->preserveWhiteSpace = false;
		$this->DDsysDoc->formatOutput = true;
		$this->DDsysDoc->load($this->DDsys); 	
	}	
	
	public function setViInfo($brief)
	{
		$this->brief = $brief;
		$this->times = $brief."_times.nc";
		$this->info = $brief."_info.nc";
		$this->info_xml = $brief."_info.xml";
		$this->cache = $brief."_cache.nc";
	}
	
	public function setViDir($ViDir)
	{
		$this->ViDir = $ViDir;
	}
7d7445ea   Elena.Budnik   THEMIS remote center 
42
43
44
45
46
 
	public function setRemoteViId($ViId)
	{
		$this->remoteViId = $ViId;
	}
34cd00b5   Elena.Budnik   new DATAMANAGER 
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
 
	public function setViId($ViId)
	{
		$this->ViId = $ViId;
	}
	
	public function setViParentsInfo($base, $mission, $instrument)
	{
		$this->base = $base;
		$this->mission = $mission;
		$this->instrument = $instrument;
	}
	
	public function setViSampling($min_sampling, $max_sampling)
	{
		$this->min_sampling = $min_sampling;
		$this->max_sampling = $max_sampling;		
	}
	
	public function setViLocation($mission_dir, $location)
	{
		$this->location = getenv("DDBASE")."/".$mission_dir."/".$location;	
		$this->ViDir = $this->location;
	}
023f1dcc   Benjamin Renard   Add NoCompression... 
70
71
72
73
74
 

	public function setViNoCompression($no_compression)
	{
		$this->noCompression = $no_compression;
	}
34cd00b5   Elena.Budnik   new DATAMANAGER 
75
76
77
78
79
 
	
	// For Remote Data Centers
	public function setViInfoFromFile($base, $ds)
	{
		$infoXml = new DomDocument("1.0");
cf197d4f   Benjamin Renard   Fix bug when DDBA... 
80
81
82
83
84
 
		$DDBASE=getenv("DDBASE");
		if (is_link($DDBASE)) {
			$DDBASE=readlink($DDBASE);
		}
		$infoXmlFile = $DDBASE."/../INFO/DDServer/$base/$ds.xml";
34cd00b5   Elena.Budnik   new DATAMANAGER 
85
86
87
88
89
90
 
		
		if (!file_exists($infoXmlFile))
				return -100;
		
		if (!$infoXml->load($infoXmlFile))
				return -10;
2f2f6cee   Elena.Budnik   FillValue by default 
91
34cd00b5   Elena.Budnik   new DATAMANAGER 
92
93
94
95
96
97
98
 
		$this->base = $base;	
		$this->mission = $infoXml->getElementsByTagName("Mission")->item(0)->nodeValue;
		$this->instrument = $infoXml->getElementsByTagName("Instrument")->item(0)->nodeValue;
		$this->min_sampling = $infoXml->getElementsByTagName("MinSampling")->item(0)->nodeValue;
		$this->remoteViId = $ds;
		$this->globalStart = $infoXml->getElementsByTagName("GlobalStart")->item(0)->nodeValue;
		$this->globalStop = $infoXml->getElementsByTagName("GlobalStop")->item(0)->nodeValue;
2f2f6cee   Elena.Budnik   FillValue by default 
99
100
 
		$fillValue = $infoXml->getElementsByTagName("FillValue");
		if ($fillValue->length > 0) $this->fillValue = $fillValue->item(0)->nodeValue;
023f1dcc   Benjamin Renard   Add NoCompression... 
101
102
 
		$noCompression = $infoXml->getElementsByTagName("NoCompression");
		if ($noCompression->length > 0) $this->noCompression = $noCompression->item(0)->nodeValue;
2f2f6cee   Elena.Budnik   FillValue by default 
103
cf197d4f   Benjamin Renard   Fix bug when DDBA... 
104
 
		$this->location = $DDBASE."/".$this->base."/".strtoupper($this->ViId)."/";
34cd00b5   Elena.Budnik   new DATAMANAGER 
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
 
		$this->ViDir = $this->location;
		//--------- Check if additional Info should be added to VI
// 		$theFormat = $infoXml->getElementsByTagName("Format")->item(0);
// 		if ($theFormat != NULL)
// 			$AddInfo = ($theFormat->nodeValue == "Text");
// 		if (!$AddInfo) 
// 			$AddInfo = ($infoXml->getElementsByTagName("AdditionalInfoNeed")->item(0) != NULL);
		
		return 0;
	}
	
	public function viExists($ViId, $baseId)
	{
		$XPath = new DOMXpath($this->DDsysDoc);
		
		$dataSet = $XPath->query("//NAME[.='$ViId']");
		
		if ($dataSet->item(0) != NULL)
		{						
			if ($dataSet->item(0)->getAttribute("base") != $baseId) return false;
			else return true;			
		}
		
		return false;	
	}
	
	public function createVi()
	{
		 $this->makeDDsys();
		 $this->makeAllInLocation();		
	}
62372d94   Elena.Budnik   RemoveVi 
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
 
	public function deleteVi($ViId, $baseId)
	{
		$XPath = new DOMXpath($this->DDsysDoc);
		
		$dataSet = $XPath->query("//NAME[.='$ViId']");
		
		if ($dataSet->item(0)->getAttribute("base") != $baseId)
			return;
		
		$ViToDelete = $dataSet->item(0)->parentNode;	
		$location = $ViToDelete->getElementsByTagName('LOCATION')->item(0)->nodeValue;
		
		$ViToDelete->parentNode->removeChild($ViToDelete);
         
		$this->DDsysDoc->save($this->DDsys);
		system("makeDDsys");
		
		foreach (glob($location."/*") as $file) unlink($file);
		rmdir($location);          
	}
34cd00b5   Elena.Budnik   new DATAMANAGER 
158
159
160
161
162
163
164
165
166
167
168
169
 
	private function makeAllInLocation()
	{
		if (!is_dir($this->location)) 
			mkdir($this->location, 0775, true);
      $currDir = getcwd();
      
		chdir($this->location);
		if (!file_exists($this->times)) system("TimesUpdate -r ".$this->times." ".$this->brief."_[0-9]*.nc");
		if (!file_exists($this->info_xml)) $this->makeInfoXml();
		if (!file_exists($this->info)) system("infoLocal2nc ".$this->info_xml." ".$this->info);
		if (!file_exists("clean")) 
		{
5f5e337d   Benjamin Renard   First step to rem... 
170
171
 
			copy(getenv("DDBASEBIN")."/Cache.template", $this->cache);         
			$SED = "sed 's/NAME/".$this->brief."/g' ".getenv("DDBASEBIN")."/Clean.template > clean";
34cd00b5   Elena.Budnik   new DATAMANAGER 
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
 
			exec($SED);
			chmod("clean", 0755);
		} 
		
		chdir($currDir);
	}
	
	private function makeInfoXml() 
	{
		$xml_dom = new DomDocument("1.0");
		$rootElement = $xml_dom->createElement("VI");
		if ($this->base != "LOCAL") 
			$rootElement->appendChild($xml_dom->createElement("DataBaseID",$this->base));
		$rootElement->appendChild($xml_dom->createElement("Mission",$this->mission));
		$rootElement->appendChild($xml_dom->createElement("Instrument",$this->instrument));
		if ($this->base != "LOCAL") 
			$rootElement->appendChild($xml_dom->createElement("DataSetID",$this->remoteViId));
		$rootElement->appendChild($xml_dom->createElement("GlobalStart", $this->globalStart));
		$rootElement->appendChild($xml_dom->createElement("GlobalStop", $this->globalStop));
		$rootElement->appendChild($xml_dom->createElement("MinSampling", $this->min_sampling));
		if ($this->max_sampling > $this->min_sampling)  
			$rootElement->appendChild($xml_dom->createElement("MaxSampling", $this->max_sampling));
2f2f6cee   Elena.Budnik   FillValue by default 
194
195
 
		if ($this->fillValue)  
			$rootElement->appendChild($xml_dom->createElement("FillValue", $this->fillValue));
023f1dcc   Benjamin Renard   Add NoCompression... 
196
197
 
		if ($this->noCompression)
			$rootElement->appendChild($xml_dom->createElement("NoCompression", $this->noCompression));
34cd00b5   Elena.Budnik   new DATAMANAGER 
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
 
		$rootElement->appendChild($xml_dom->createElement("LocalStart"));
		$rootElement->appendChild($xml_dom->createElement("LocalStop"));
		$xml_dom->appendChild($rootElement);
		$xml_dom->save($this->location."/".$this->info_xml);		
	}

	private function makeDDsys()
	{
		$newNode = $this->DDsysDoc->createElement("VI");
		$name = $newNode->appendChild($this->DDsysDoc->createElement("NAME",$this->ViId));
		$name->setAttribute('base', $this->base);
		$name->setAttribute('mission', $this->mission);
		$name->setAttribute('instrument', $this->instrument);

		if (substr($this->location, -1) !== "/") 
			$newNode->appendChild($this->DDsysDoc->createElement("LOCATION",$this->location."/")); 
		else 
			$newNode->appendChild($this->DDsysDoc->createElement("LOCATION",$this->location));

		$newNode->appendChild($this->DDsysDoc->createElement("TIMES", $this->times));
		$newNode->appendChild($this->DDsysDoc->createElement("INFO", $this->info));
		$newNode->appendChild($this->DDsysDoc->createElement("CACHE", $this->cache));

		$this->DDsysDoc->documentElement->appendChild($newNode);
			
		$this->DDsysDoc->save($this->DDsys);
		system("makeDDsys");  	
	}
62372d94   Elena.Budnik   RemoveVi 
226
34cd00b5   Elena.Budnik   new DATAMANAGER 
227
228
229
230
231
232
233
234
235
 
// 	public function getRemoteLocation($base, $remoteVi)
// 	{
// 		$XPath = new DOMXpath($this->DDsysDoc);
// 		
// 		$dataSet = $XPath->query("//NAME[.='$ViId']");
// 		$allVis = $this->DDsysDoc-	
// 	
//	}
162f5e88   Elena.Budnik   immediate VI lock 
236
 
	public function lockVi()
34cd00b5   Elena.Budnik   new DATAMANAGER 
237
 
	{
c8f4ea1d   Elena.Budnik   new cdf2nc, updat... 
238
 
		//  Stamp -> this directory is being updated
34cd00b5   Elena.Budnik   new DATAMANAGER 
239
240
241
242
243
 
		touch($this->ViDir."/LOCK");
  
		//	fprintf($STDERR,$ViDir." is LOCKED\n");
	}
162f5e88   Elena.Budnik   immediate VI lock 
244
 
	public function unlockVi()
34cd00b5   Elena.Budnik   new DATAMANAGER 
245
246
247
248
249
 
	{
		if (file_exists($this->ViDir."/LOCK")) unlink($this->ViDir."/LOCK");  
		//	fprintf($STDERR,$ViDir." is UNLOCKED\n");
	}
7d7445ea   Elena.Budnik   THEMIS remote center 
250
 
	public function addRemoteData($id, $ncFiles, $start, $stop, $notFirst = true)
34cd00b5   Elena.Budnik   new DATAMANAGER 
251
 
	{
34cd00b5   Elena.Budnik   new DATAMANAGER 
252
253
254
 
		$WORKING_DIR = getcwd();
		chdir($this->ViDir);
		system("./clean");
2c71cf83   Elena.Budnik   THEMIS update sta... 
255
34cd00b5   Elena.Budnik   new DATAMANAGER 
256
257
258
 
		foreach ($ncFiles as $ncFile)
		{
			//TODO errors
f8a71629   Elena.Budnik   cut long ViId 
259
260
261
 
			//if (!file_exists($ncFile))
			rename($WORKING_DIR."/".$ncFile,$this->ViDir."/".$ncFile);
			system("TimesUpdate -u ".$this->times." ".$ncFile);
34cd00b5   Elena.Budnik   new DATAMANAGER 
262
263
 
		}
7d7445ea   Elena.Budnik   THEMIS remote center 
264
265
266
 
		if ($notFirst)
				system("TimesUpdateNoData ".$this->times." ".$start." ".$stop);
34cd00b5   Elena.Budnik   new DATAMANAGER 
267
 
		chdir($WORKING_DIR);
34cd00b5   Elena.Budnik   new DATAMANAGER 
268
 
	}
0ec21281   Elena.Budnik   reorganization + ... 
269
270
271
 
	
	public function setInfo($infoFile)
	{
cf197d4f   Benjamin Renard   Fix bug when DDBA... 
272
273
274
275
276
 
		$DDBASE=getenv("DDBASE");
		if (is_link($DDBASE)) {
			$DDBASE=readlink($DDBASE);
		}
		$fullInfoName = $DDBASE."/../INFO/bases/".$this->base."/$infoFile";
0ec21281   Elena.Budnik   reorganization + ... 
277
278
279
280
281
 
	 	 
		if (file_exists($fullInfoName)) unlink($fullInfoName);
		//TODO errors		
			//if (!file_exists($infoFile))	
		$WORKING_DIR = getcwd();
0ec21281   Elena.Budnik   reorganization + ... 
282
 
		rename("$WORKING_DIR/$infoFile", $fullInfoName);
0ec21281   Elena.Budnik   reorganization + ... 
283
 
	}
c8f4ea1d   Elena.Budnik   new cdf2nc, updat... 
284
285
286
 
	
	public function updateRemoteStart()
	{
1faf036d   Elena.Budnik   bug in UpdateRemo... 
287
 
		$startStamp = shell_exec("GetStartTime ".$this->location.$this->times);
d63eef4f   Elena.Budnik   typo 
288
 
		$startIso = date("Y-m-d\TH:i:s", intval($startStamp)).substr(number_format(fmod(floatval($startStamp),1),3, '.', ''),1,4)."Z";
c8f4ea1d   Elena.Budnik   new cdf2nc, updat... 
289
290
291
292
293
294
295
296
297
298
299
300
301
 
		
		$xml_dom = new DomDocument("1.0");
		$xml_dom->load($this->location."/".$this->info_xml);
		$startNode = $xml_dom->getElementsByTagName("GlobalStart")->item(0);
		$startNode->nodeValue = $startIso;	
		
		$xml_dom->save($this->location."/".$this->info_xml);
		
		$currDir = getcwd();		
		chdir($this->location);		
		system("infoLocal2nc ".$this->info_xml." ".$this->info);
		chdir($currDir);
	}
1bd736dd   Elena.Budnik   add cleanRemote 
302
d63eef4f   Elena.Budnik   typo 
303
 
	public function updateViStop($globalStop)
2c71cf83   Elena.Budnik   THEMIS update sta... 
304
305
306
 
	{
		$xml_dom = new DomDocument("1.0");
		$xml_dom->load($this->location."/".$this->info_xml);
4a24f977   Elena.Budnik   typo 
307
2c71cf83   Elena.Budnik   THEMIS update sta... 
308
 
		$stopNode = $xml_dom->getElementsByTagName("GlobalStop")->item(0);
4a24f977   Elena.Budnik   typo 
309
 
		$stopNode->nodeValue = $globalStop;
d63eef4f   Elena.Budnik   typo 
310
 
        $xml_dom->save($this->location."/".$this->info_xml);
2c71cf83   Elena.Budnik   THEMIS update sta... 
311
312
313
 
		
		$currDir = getcwd();		
		chdir($this->location);
4a24f977   Elena.Budnik   typo 
314
2c71cf83   Elena.Budnik   THEMIS update sta... 
315
316
317
318
319
 
		system("infoLocal2nc ".$this->info_xml." ".$this->info);
		chdir($currDir);
	}
1bd736dd   Elena.Budnik   add cleanRemote 
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
 
	public function cleanRemote($opt = NULL)
	{
		$res_before = disk_free_space(getenv("DDBASE"));
		$Days = 10;  // default
		$SecDay = 3600*24;
		$nodata = false;

		if (isset($opt)) {
			if ($opt == "NODATA") 
					$nodata = true;
			else {
				$Days = is_numeric($opt) ? $opt : 10;
				settype($Days, "int");
				echo "!!!!!!!!!!!!!!!! Days   ".$Days.PHP_EOL;
			}
		}
		
		$GlobalDelta = $Days*$SecDay;
	
		$xp = new domxpath($this->DDsysDoc);
		$dataSets = $xp->query("//NAME[attribute::base!='LOCAL']");
		
		for ($i = 0; $i < $dataSets->length; $i++) 
		{ 
			$parent = $dataSets->item($i)->parentNode;
			$location = $parent->getElementsByTagName("LOCATION")->item(0)->nodeValue;
 
			if (!is_dir($location)) {
				$parent->parentNode->removeChild($parent);
			}
			else {
				$time_info = $parent->getElementsByTagName("TIMES")->item(0)->nodeValue;
				chdir($location);
				system("./clean");
				touch($location."/LOCK");

				fprintf(STDERR,"In the ".$location."\n");

				/*  Delete No Data Intervals */        
				if ($nodata) {
					system("CleanNoData ".$time_info); 
				}
				/*  Delete Old Files */  
				else {
21ee78c5   Elena.Budnik   always keeps firs... 
364
 
                    $isFirst = true;
1bd736dd   Elena.Budnik   add cleanRemote 
365
 
					foreach (glob("*.gz") as $filename) {
21ee78c5   Elena.Budnik   always keeps firs... 
366
367
368
369
 
                        if ($isFirst) { // we should always keep first file
                            touch($filename);
                            $isFirst = false;
                        }
1bd736dd   Elena.Budnik   add cleanRemote 
370
371
 
						$info = stat($filename);
						if ((time() - $info['atime']) > $GlobalDelta) {
21ee78c5   Elena.Budnik   always keeps firs... 
372
 
						//	fprintf(STDERR, $filename."\n");
1bd736dd   Elena.Budnik   add cleanRemote 
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
 
							system("TimesUpdate -d ".$time_info." ".$filename);
						}
						//fprintf(STDERR, date("Y-m-d",$info['atime'])." - ".date("Y-m-d",$info['mtime'])."\n");
					}
				}

			system("DDClearTimes ".$time_info);
			unlink($location."/LOCK");  
        }
		}
    /* 
		$res_after = disk_free_space(getenv("DDBASE"));
		$difference = ($res_after - $res_before)/1024.0;
		$res_after /=  1024.0;
		fprintf(STDERR,"+ Free Space  ".$difference." kB\n");
		fprintf(STDERR,"  Free Space  ".$res_after." kB\n");*/
		
		$this->DDsysDoc->save($this->DDsys);
		system("makeDDsys");	
	}
88a88fcc   Benjamin Renard   Add script to con... 
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
 

	function convertVIDDTimeToDouble($ViId) {
		$XPath = new DOMXpath($this->DDsysDoc);

		$dataSet = $XPath->query("//NAME[.='$ViId']");

		if ($dataSet->length == 0) {
			echo "[ERROR] NO $ViId in DDsys.xml".PHP_EOL;
			return FALSE;
		}

		$dataSet = $dataSet->item(0);

		$locationNodes = $dataSet->parentNode->getElementsByTagName('LOCATION');

		if ($locationNodes->length == 0) {
			echo "[ERROR] Cannot retrieve LOCATION for $ViId in DDsys.xml".PHP_EOL;
			return FALSE;
		}

		$location = $locationNodes->item(0)->nodeValue;


		if (file_exists($location."/LOCK")) {
			echo "[ERROR] VI $ViId already locked by another process".PHP_EOL;
			return FALSE;
		}

		echo "[INFO] Lock VI $ViId".PHP_EOL;
		touch($location."/LOCK");

		$WORKING_DIR = getcwd();
		chdir($location);

		echo "[INFO] Cleanup VI $ViId".PHP_EOL;
		system("./clean");
		echo "[INFO] Convert data files".PHP_EOL;
		system("ConvertDDTimeToDouble *.nc.gz");

		unlink($location."/LOCK");
		
	}
34cd00b5   Elena.Budnik   new DATAMANAGER 
435
436
 
} 
?>
5f5e337d   Benjamin Renard   First step to rem... 
437