Fix scripts for HAPI & EPN-TAP (cf. #11867)

Myriam Bouchemit
1 parent fa5a398d
Showing 2 changed files with 74 additions and 28 deletions Show diff stats
update_amda/makeEPNTAPGranules.php
update_amda/makeHAPIMetadata.php
@@ -41,7 +41,7 @@ $mapping = array(
         'c2_resol_max' => '',
         'c3_resol_min' => '',
         'c3_resol_max' => '',
-        'spatial_frame_type' => '',
+        'spatial_frame_type' => 'none',
         'incidence_min' => '',
         'incidence_max' => '',
         'emergence_min' => '',
@@ -55,7 +55,7 @@ $mapping = array(
         'creation_date' => '@@creation_date@@',
         'modification_date' => '@@modification_date@@',
         'release_date' => '@@release_date@@',
-        'service_title' => 'AMDADB',
+        'service_title' => 'amdadb',
         'access_url' => '@@amda_rest_api_entry_point@@getGranule.php?obs_id=@@dataset_id@@&start=@@granule_start_time_timestamp@@&stop=@@granule_stop_time_timestamp@@',
         'access_format' => 'application/x-cdf-istp',
         'target_region' => '@@target_region@@',
@@ -86,14 +86,15 @@ function parseSampling($sampling) {
 	}
 }
-function parseProcessingLevel($info) {
-	if (strpos($info, 'L1') !== FALSE) {
-		return '2';
-	}
-	if (strpos($info, 'L3') !== FALSE) {
-		return '5';
+function parseProcessingLevel($processing_level) {
+	switch ($processing_level) {
+		case 'Uncalibrated':
+			return '2';
+		case 'ValueAdded':
+			return '5';
+		default:
+			return '3';
 	}
-	return '3';
 }
 function getEpnTapMeasurementType($measurement_type, $UCDs) {
@@ -104,14 +105,14 @@ function getEpnTapMeasurementType($measurement_type, $UCDs) {
 			break;
 		case 'IonComposition':
 		case 'NeutralGas':
-			$generic_ucd = 'phys.composition';
+			$generic_ucd = 'phys';
 			break;
 		case 'EnergeticParticles':
-			$generic_ucd = 'phys.particle';
+			$generic_ucd = 'phys';
 			break;
 		case 'Waves':
 		case 'Waves.Passive':
-			$generic_ucd = 'em.pw';
+			$generic_ucd = 'phys';
 			break;
 		case 'Ephemeris':
 			$generic_ucd = 'pos.ephem';
@@ -138,9 +139,10 @@ function getEpnTapMeasurementType($measurement_type, $UCDs) {
 			echo "[WARNING] UCD - Unknown measurement type: ".$measurement_type.PHP_EOL;
 			break;
 	}
-	if (!empty($generic_ucd) && !in_array($generic_ucd, $UCDs))
+	if (!empty($generic_ucd) && !in_array($generic_ucd, $UCDs)) {
 		array_unshift($UCDs, $generic_ucd);
-	return implode(';', $UCDs);
+	}
+	return implode('#', $UCDs);
 }
 function timestampToJulianDay($timestamp) {
@@ -161,6 +163,7 @@ function getTargetClass($spaseRegion) {
 		case "Asteroid":
 			return "asteroid";
 		case "Comet":
+		case "Rosetta@C-G":
 			return "comet";
 		case "Earth":
 		case "Jupiter":
@@ -180,6 +183,12 @@ function getTargetClass($spaseRegion) {
 			return "interplanetary_medium";
 		case "Sun":
 			return "star";
+		case "Bepi":
+		case "Juno":
+		case "SolO":
+		case "PSP":
+		case "Cassini":
+			return "spacecraft";
 	}
 	echo "[WARNING] Error in ObservatoryRegion definition : ".$spaseRegion.PHP_EOL;
 	return "planet";
@@ -203,6 +212,7 @@ function getTargetName($targetClass, $spaseRegion, $missionName, $dataset_id) {
 		case "comet":
 			switch ($missionName) {
 				case "Rosetta":
+				case "Rosetta@C-G":
 					return "67P";
 				case "Giotto":
 					return "1P";
@@ -213,9 +223,21 @@ function getTargetName($targetClass, $spaseRegion, $missionName, $dataset_id) {
 				case "ICE":
 					if ($dataset_id == "ice-mag-p21")
 						return "21P";
+				case "Solar Wind Propagation Models":
+					if (in_array($dataset_id, array("bls-ros-omni", "tao-ros-sw")))
+						return "67P";
 			}
-			echo "[WARNING] Comet Id not defined for mission : ".$missionName.PHP_EOL;
+			echo "[WARNING] Comet Id not defined for mission : ".$missionName." - ".$dataset_id.PHP_EOL;
 			return "";
+		case "spacecraft":
+			switch ($spaseRegion) {
+				case "SolO":
+					return "SolarOrbiter";
+				case "Bepi":
+					return "BepiColombo";
+				default:
+					return $spaseRegion;
+			}
 	}
 	echo "[WARNING] Cannot retrieve target name from target class : ".$targetClass.PHP_EOL;
 	return "";
@@ -276,13 +298,13 @@ $xpath = new DOMXpath($doc);
 $output_file_handle = fopen($output_file, 'w');
 fputcsv($output_file_handle, array_keys($mapping));
-$dataset_nodes = $xpath->query("/dataRoot/dataCenter/mission/instrument/dataset | /dataRoot/dataCenter/mission/observatory/instrument/dataset | /dataRoot/dataCenter/mission/instrument/datasetGroup/dataset");
+$dataset_nodes = $xpath->query("/dataRoot/dataCenter/mission/instrument/dataset | /dataRoot/dataCenter/mission/observatory/instrument/dataset | /dataRoot/dataCenter/mission/instrument/datasetGroup/dataset | /dataRoot/dataCenter/mission/observatory/instrument/datasetGroup/dataset");
 $datasets = array();
 foreach ($dataset_nodes as $dataset_node) {
 	$dataset_place_holders = array();
 	//Dataset id
 	$datasetId = $dataset_node->getAttribute("xml:id");
-	//if ($datasetId != 'psp-het-1hr')
+	//if ($datasetId != 'so-pas-mom')
 	//	continue;
 	if (empty($datasetId)) {
 		echo "[WARNING] Cannot retrieve dataset id".PHP_EOL;
@@ -293,11 +315,17 @@ foreach ($dataset_nodes as $dataset_node) {
 	//Exclude dataset with group
 	$to_exclude = FALSE;
 	$crt_node = $dataset_node;
+	$time_restriction = 0;
 	while (($crt_node != NULL) && !$to_exclude) {
 		if ($crt_node->nodeType == XML_ELEMENT_NODE) {
 			$group = $crt_node->getAttribute("group");
 			if (!empty($group)) {
-				$to_exclude = TRUE;
+				$restriction = $crt_node->getAttribute("restriction");
+				if (empty($restriction))
+					$to_exclude = TRUE;
+				$time_restriction = strtotime($restriction);
+				if ($time_restriction === FALSE)
+					$to_exclude = TRUE;
 			}
 		}
 		$crt_node = $crt_node->parentNode;
@@ -313,22 +341,28 @@ foreach ($dataset_nodes as $dataset_node) {
 		$ucd = trim($parameter_node->getAttribute('ucd'));
 		if (empty($ucd))
 			continue;
-		$ucd_list = explode(';', $ucd);
+		if (!in_array($ucd, $UCDs)) {
+			$UCDs[] = $ucd;
+		}
+		/*$ucd_list = explode(';', $ucd);
 		foreach ($ucd_list as $u) {
 			$u = trim($u);
 			if (empty($u))
 				continue;
 			if (!in_array($u, $UCDs))
 				$UCDs[] = $u;
-		}
+		}*/
 	}
 	$dataset_place_holders['min_sampling'] = parseSampling($dataset_node->getAttribute("sampling"));
 	$dataset_place_holders['max_sampling'] = parseSampling($dataset_node->getAttribute("maxSampling"));
+	if (empty($dataset_place_holders['max_sampling'])) {
+		$dataset_place_holders['max_sampling'] = $dataset_place_holders['min_sampling'];
+	}
 	$dataset_place_holders['spase_resource_id'] = $dataset_node->getAttribute("spaseId");
 	$dataset_place_holders['spase_measurement_type'] = $dataset_node->getAttribute("measurement_type");
 	$dataset_place_holders['measurement_type'] = getEpnTapMeasurementType($dataset_place_holders['spase_measurement_type'], $UCDs);
-	$dataset_place_holders['processing_level'] = parseProcessingLevel($dataset_node->getAttribute("xml:id"));
+	$dataset_place_holders['processing_level'] = parseProcessingLevel($dataset_node->getAttribute("processing_level"));
 	$target = $dataset_node->getAttribute("target");
@@ -384,13 +418,16 @@ foreach ($dataset_nodes as $dataset_node) {
 		$mission_name = $observatory_node->getAttribute('name');
 	}
 	$dataset_place_holders['mission_name'] = utf8_encode($mission_name);
+	if ($dataset_node->getAttribute('simulation') == '1') {
+		$dataset_place_holders['mission_name'] = 'Simulation';
+	}
 	$dataset_place_holders['instrument_name'] = preg_replace('/[^:\/\(\) \w-]/', ' ', $instrument_name);
 	if ($dataset_place_holders['instrument_name'] != $instrument_name) {
-		echo "=================> ".$dataset_place_holders['instrument_name'].PHP_EOL;
-	}
-	if (!empty($dataset_group_node)) {
-		if ($dataset_place_holders['processing_level'] == 'L2') {
-			$dataset_place_holders['processing_level'] = parseProcessingLevel($dataset_group_node->getAttribute("xml:id"));
+		if ($dataset_place_holders['instrument_name'] == "IS   IS EPI-Lo") {
+			$dataset_place_holders['instrument_name'] = "ISOIS EPI-Lo";
+		}
+		else if ($dataset_place_holders['instrument_name'] == "IS   IS EPI-Hi") {
+			$dataset_place_holders['instrument_name'] = "ISOIS EPI-Hi";
 		}
 	}
@@ -424,6 +461,16 @@ foreach ($dataset_nodes as $dataset_node) {
 		$granule_place_holders['granule_index'] = str_replace(".nc", "", $granule_info[2]);
 		$granule_place_holders['granule_start_time'] = timestampToJulianDay($granule_info[0]);
 		$granule_place_holders['granule_stop_time']  = timestampToJulianDay($granule_info[1]);
+		if ($granule_place_holders['granule_start_time'] > $granule_place_holders['granule_stop_time']) {
+			echo "[ERROR] Bad start/stop definition for granule ".$granule_place_holders['granule_index']." of ".$datasetId.PHP_EOL;
+			continue;
+		}
+		if (!empty($time_restriction)) {
+			// Exclude restricted granules
+			if (intval($granule_info[1]) > $time_restriction) {
+				continue;
+			}
+		}
 		$granule_place_holders['modification_date']  = timestampToISO($granule_info[3]);
 		$granule_place_holders['granule_start_time_timestamp'] = $granule_info[0];
 		$granule_place_holders['granule_stop_time_timestamp']  = $granule_info[1];
@@ -9,10 +9,9 @@ $output_file      = getenv(&#39;HAPIData&#39;).&quot;/metadata/amda-catalog.json&quot;;
 $doc = new DOMDocument();
 @$doc->load($localparams_tree);
-
 $xpath = new DOMXpath($doc);
-$dataset_nodes = $xpath->query("/dataRoot/dataCenter/mission/instrument/dataset | /dataRoot/dataCenter/mission/observatory/instrument/dataset | /dataRoot/dataCenter/mission/instrument/datasetGroup/dataset");
+$dataset_nodes = $xpath->query("/dataRoot/dataCenter/mission/instrument/dataset | /dataRoot/dataCenter/mission/observatory/instrument/dataset | /dataRoot/dataCenter/mission/instrument/datasetGroup/dataset | /dataRoot/dataCenter/mission/observatory/instrument/datasetGroup/dataset");
 $datasets = array();
 foreach ($dataset_nodes as $dataset_node) {
 	//Dataset id