Alexandre Schulz / amda_data_generator

Commit b79dbb0f565fd82c1dc5a495a45c4ad0e253b175

Authored by Benjamin Renard
1 parent 8c2373ac
Exists in master

New version for JUICE/JDC

Showing 3 changed files with 146 additions and 123 deletions   Show diff stats
Inline Side-by-side
scripts/jdc_constant.cdl
  Diff comments   View file @b79dbb0
1   -netcdf jdc_constant {
  1 +netcdf jdc_constant_reorder {
2 2 dimensions:
3   - dim12 = 12 ;
4   - dim72 = 72 ;
5   -
  3 + dim6 = 6 ;
  4 + dim12 = 12 ;
  5 + dim25 = 25 ;
  6 + dim75 = 75 ;
6 7 variables:
7   - float ELEVATION_P0(dim12) ;
8   - float ENERGY_P0(dim72) ;
9   -
  8 + float ELEVATION_12(dim12) ;
  9 + float ELEVATION_6(dim6) ;
  10 + float ENERGY_75(dim75) ;
  11 + float ENERGY_25(dim25) ;
10 12 data:
11 13  
12   - ELEVATION_P0 =
13   - 3.0,7.0,13.0,19.0,26.0,32.0,38.0,45.0,52.0,60.0,68.0,75.0;
14   -
15   - ENERGY_P0 =
16   - 1.0, 3.3, 5.87, 16.16, 18.74, 21.31, 34.13, 38.38, 43.16, 69.0,
17   - 77.59, 87.25, 139.49, 156.85, 176.37, 281.97, 317.06, 356.52, 569.97,
18   - 640.9, 720.66, 1152.11, 1295.49, 1456.71, 2328.77, 2618.57, 2944.43,
19   - 4707.06, 5292.8, 5951.43, 9513.99, 10697.87, 12029.06, 19229.45, 21622.2,
20   - 24312.67, 34564.44, 30739.55, 27337.91, 17101.47, 15208.97, 13525.89,
21   - 8461.12, 7524.76, 6692.02, 4186.13, 3722.85, 3310.85, 2071.04, 1841.83,
22   - 1637.99, 1024.6, 911.2, 810.35, 506.88, 450.78, 400.89, 250.76, 223.0,
23   - 198.32, 124.05, 110.32, 98.11, 61.36, 54.57, 48.53, 30.36, 26.99, 24.01,
24   - 13.59, 11.02, 8.44;
  14 + ELEVATION_12 = 3, 7, 13, 19, 26, 32, 38, 45, 52, 60, 68, 75 ;
  15 +
  16 + ELEVATION_6 = 5, 16, 29, 41.5, 56, 71.5 ;
  17 +
  18 + ENERGY_75 = 0.10, 0.10, 0.10, 1, 3.3, 5.87, 8.44, 11.02, 13.59, 16.16, 18.74, 21.31, 24.01,
  19 + 26.99, 30.36, 34.13, 38.38, 43.16, 48.53, 54.57, 61.36, 69, 77.59, 87.25,
  20 + 98.11, 110.32, 124.05, 139.49, 156.85, 176.37, 198.32, 223, 250.76,
  21 + 281.97, 317.06, 356.52, 400.89, 450.78, 506.88, 569.97, 640.9, 720.66,
  22 + 810.35, 911.2, 1024.6, 1152.11, 1295.49, 1456.71, 1637.99, 1841.83,
  23 + 2071.04, 2328.77, 2618.57, 2944.43, 3310.85, 3722.85, 4186.13, 4707.06,
  24 + 5292.8, 5951.43, 6692.02, 7524.76, 8461.12, 9513.99, 10697.87, 12029.06,
  25 + 13525.89, 15208.97, 17101.47, 19229.45, 21622.2, 24312.67, 27337.91,
  26 + 30739.55, 34564.44 ;
25 27  
  28 + ENERGY_25 = 0.1, 3.3, 11.02, 18.74, 26.99, 38.38, 54.57, 77.59, 110.32, 156.85,
  29 + 223, 317.06, 450.78, 640.9, 911.2, 1295.49, 1841.83, 2618.57, 3722.85,
  30 + 5292.8, 7524.76, 10697.87, 15208.97, 21622.2, 30739.55 ;
26 31 }
... ...
scripts/jdc_constant.nc
View file @b79dbb0
No preview for this file type
scripts/juice_jdc.py
  Diff comments   View file @b79dbb0
... ... @@ -7,20 +7,57 @@ from amda_data_generator.get import LocalGetter
7 7 from amda_data_generator.convert.netcdf import NetCDF2NetCDF
8 8  
9 9 from netCDF4 import Dataset
  10 +import numpy as np
10 11  
11 12 input_files = '/home/brenard/JDC_DATA/AMDA/'
12 13  
  14 +reorder_energy = True
  15 +
  16 +elevation = [3.0, 7.0, 13.0, 19.0, 26.0, 32.0, 38.0, 45.0, 52.0, 60.0, 68.0, 75.0]
  17 +elevation = np.array(elevation)
  18 +
  19 +energy = [
  20 + 0.1, 0.1, 0.1, 8.44, 11.02, 13.59, 24.01, 26.99, 30.36, 48.53, 54.57, 61.36, 98.11, 110.32, 124.05, 198.32,
  21 + 223.00, 250.76, 400.89, 450.78, 506.88, 810.35, 911.20, 1024.60, 1637.99, 1841.83, 2071.04,
  22 + 3310.85, 3722.85, 4186.13, 6692.02, 7524.76, 8461.12, 13525.89, 15208.97, 17101.47, 27337.91,
  23 + 30739.55, 34564.44, 24312.67, 21622.20, 19229.45, 12029.06, 10697.87, 9513.99, 5951.43, 5292.80,
  24 + 4707.06, 2944.43, 2618.57, 2328.77, 1456.71, 1295.49, 1152.11, 720.66, 640.90, 569.97, 356.52,
  25 + 317.06, 281.97, 176.37, 156.85, 139.49, 87.25, 77.59, 69.00, 43.16, 38.38, 34.13, 21.31, 18.74,
  26 + 16.16, 5.87, 3.30, 1.00
  27 +]
  28 +energy = np.array(energy)
  29 +energy = energy[::-1] # Reverse order
  30 +
  31 +
  32 +elevation_12 = elevation
  33 +elevation_6 = [(elevation_12[2*i+1]+elevation_12[2*i])/2 for i in range(6)]
13 34  
14   -def jdc_get_time(x):
15   - return (datetime(year=2000, month=1, day=1, tzinfo=timezone.utc) + timedelta(days=x)).timestamp()
16 35  
  36 +energy_75 = energy
  37 +energy_25 = energy[1::3]
17 38  
18   -def jdc_extract_spectro(real_var):
19   - return real_var[:, :, 0:72]
  39 +if reorder_energy:
  40 + energy_75_sort = np.argsort(energy_75)
  41 + energy_75 = energy_75[energy_75_sort]
  42 +
  43 + energy_25_sort = np.argsort(energy_25)
  44 + energy_25 = energy_25[energy_25_sort]
  45 +
  46 +np.set_printoptions(suppress=True)
  47 +print(", ".join(np.char.mod('%.2f', energy_75)))
  48 +print(", ".join(np.char.mod('%.2f', energy_25)))
  49 +
  50 +#exit(0)
  51 +
  52 +
  53 +def jdc_get_time(x):
  54 + return (datetime(year=2000, month=1, day=1, tzinfo=timezone.utc) + timedelta(days=x)).timestamp()
20 55  
21 56  
22   -def jdc_extract_backgrd(real_var, args):
23   - return real_var[:, :, args["index"]]
  57 +def jdc_extract_spectro(real_var, args):
  58 + if reorder_energy:
  59 + return real_var[:, :, args["order"]]
  60 + return real_var
24 61  
25 62  
26 63 datasets_info = {
... ... @@ -41,120 +78,89 @@ datasets_info = {
41 78 },
42 79 "JDC_SCIENCE": {
43 80 "prefix": "jdc",
44   - "groups": ["P0", "P4"],
45   - "mapping": {
  81 + "groups": {
46 82 "P0": {
47   - "Time": ("time", jdc_get_time),
48   - "start": ("start", jdc_extract_spectro),
49   - "stop": ("stop", jdc_extract_spectro),
50   - "start_backgrd_e72": ("start", jdc_extract_backgrd, {"index": 72}),
51   - "start_backgrd_e73": ("start", jdc_extract_backgrd, {"index": 73}),
52   - "start_backgrd_e74": ("start", jdc_extract_backgrd, {"index": 74}),
53   - "stop_backgrd_e72": ("stop", jdc_extract_backgrd, {"index": 72}),
54   - "stop_backgrd_e73": ("stop", jdc_extract_backgrd, {"index": 73}),
55   - "stop_backgrd_e74": ("stop", jdc_extract_backgrd, {"index": 74}),
  83 + "direction": 12,
  84 + "azimuth" : 16,
  85 + "energy" : 75,
  86 + "stops" : 4
  87 + },
  88 + "P1": {
  89 + "direction": 6,
  90 + "azimuth" : 16,
  91 + "energy" : 75,
  92 + "stops" : 4
  93 + },
  94 + "P2": {
  95 + "direction": 6,
  96 + "azimuth" : 8,
  97 + "energy" : 25,
  98 + "stops" : 4
  99 + },
  100 + "P3": {
  101 + "direction": 6,
  102 + "azimuth" : 8,
  103 + "energy" : 25,
  104 + "stops" : 4
56 105 },
57 106 "P4": {
58   - "Time": ("time", jdc_get_time),
59   - "start": "start",
60   - "stop": "stop",
  107 + "direction": 6,
  108 + "azimuth" : 8,
  109 + "energy" : 25,
  110 + "stops" : 4
  111 + },
  112 + "N1": {
  113 + "direction": 6,
  114 + "azimuth" : 16,
  115 + "energy" : 75,
  116 + "stops" : 4
  117 + },
  118 + "N2": {
  119 + "direction": 6,
  120 + "azimuth" : 8,
  121 + "energy" : 25,
  122 + "stops" : 4
  123 + },
  124 + "N3": {
  125 + "direction": 6,
  126 + "azimuth" : 8,
  127 + "energy" : 25,
  128 + "stops" : 4
61 129 }
62 130 }
63 131 }
64 132 }
65 133  
66   -elevation_p0 = [3.0, 7.0, 13.0, 19.0, 26.0, 32.0, 38.0, 45.0, 52.0, 60.0, 68.0, 75.0]
67   -
68   -# This table is used in reverse order:
69   -energy_p0 = [
70   - 8.44,
71   - 11.02,
72   - 13.59,
73   - 24.01,
74   - 26.99,
75   - 30.36,
76   - 48.53,
77   - 54.57,
78   - 61.36,
79   - 98.11,
80   - 110.32,
81   - 124.05,
82   - 198.32,
83   - 223.00,
84   - 250.76,
85   - 400.89,
86   - 450.78,
87   - 506.88,
88   - 810.35,
89   - 911.20,
90   - 1024.60,
91   - 1637.99,
92   - 1841.83,
93   - 2071.04,
94   - 3310.85,
95   - 3722.85,
96   - 4186.13,
97   - 6692.02,
98   - 7524.76,
99   - 8461.12,
100   - 13525.89,
101   - 15208.97,
102   - 17101.47,
103   - 27337.91,
104   - 30739.55,
105   - 34564.44,
106   - 24312.67,
107   - 21622.20,
108   - 19229.45,
109   - 12029.06,
110   - 10697.87,
111   - 9513.99,
112   - 5951.43,
113   - 5292.80,
114   - 4707.06,
115   - 2944.43,
116   - 2618.57,
117   - 2328.77,
118   - 1456.71,
119   - 1295.49,
120   - 1152.11,
121   - 720.66,
122   - 640.90,
123   - 569.97,
124   - 356.52,
125   - 317.06,
126   - 281.97,
127   - 176.37,
128   - 156.85,
129   - 139.49,
130   - 87.25,
131   - 77.59,
132   - 69.00,
133   - 43.16,
134   - 38.38,
135   - 34.13,
136   - 21.31,
137   - 18.74,
138   - 16.16,
139   - 5.87,
140   - 3.30,
141   - 1.00,
142   -]
143   -
144   -
145 134 local_navigator = LocalFilesNavigator(input_files)
146 135 local_getter = LocalGetter()
147 136 converter = NetCDF2NetCDF()
148 137  
  138 +constants = {}
  139 +dimensions = {}
  140 +for group in datasets_info["JDC_SCIENCE"]["groups"]:
  141 + if datasets_info["JDC_SCIENCE"]["groups"][group]["direction"] == 12:
  142 + elev = elevation_12
  143 + else:
  144 + elev = elevation_6
  145 + if datasets_info["JDC_SCIENCE"]["groups"][group]["energy"] == 75:
  146 + en = energy_75
  147 + else:
  148 + elev = energy_25
  149 + constants["ELEVATION_{}".format(group)] = elev
  150 + constants["ENERGY_{}".format(group)] = en
  151 +
  152 +
  153 +
149 154 for dataset_key in datasets_info:
150 155 for item in local_navigator.iter(extension='nc', recursive=True,
151   - name_regex=re.compile(r'[0-9-]*_pfm_999_{}'.format(dataset_key))):
  156 + name_regex=re.compile(r'[a-z0-9-_]*_pfm_[0-9]*_{}'.format(dataset_key))):
  157 + print("=> {}".format(item.filename))
152 158 input_data = local_getter.get(item).get_data(binary=True)
153 159  
154 160 groups = []
155 161 nc_groups = []
156 162 if 'groups' in datasets_info[dataset_key]:
157   - groups = datasets_info[dataset_key]["groups"]
  163 + groups = datasets_info[dataset_key]["groups"].keys()
158 164 nc = Dataset('in-mem-file', memory=input_data, mode="r")
159 165 nc_groups = nc.groups
160 166 nc.close()
... ... @@ -168,7 +174,19 @@ for dataset_key in datasets_info:
168 174 print("[WARNING] Missing group {} in {}".format(group, item.filename))
169 175 continue
170 176 target_nc_dir += ("_" + group)
171   - mapping = datasets_info[dataset_key]['mapping'][group]
  177 + group_info = datasets_info[dataset_key]["groups"][group]
  178 + if group_info["energy"] == 75:
  179 + mapping = {
  180 + "Time": ("time", jdc_get_time),
  181 + "start": ("start", jdc_extract_spectro, {"order": energy_75_sort}),
  182 + "stop": ("stop", jdc_extract_spectro, {"order": energy_75_sort}),
  183 + }
  184 + else:
  185 + mapping = {
  186 + "Time": ("time", jdc_get_time),
  187 + "start": ("start", jdc_extract_spectro, {"order": energy_25_sort}),
  188 + "stop": ("stop", jdc_extract_spectro, {"order": energy_25_sort}),
  189 + }
172 190 else:
173 191 mapping = datasets_info[dataset_key]['mapping']
174 192 if not os.path.exists(target_nc_dir):
... ... @@ -183,4 +201,4 @@ for dataset_key in datasets_info:
183 201 group=group,
184 202 mapping=mapping, time_as_double=False)
185 203  
186   -print(list(reversed(energy_p0))) # reverse
187 204 \ No newline at end of file
  205 +#print(list(reversed(energy_p0))) # reverse
188 206 \ No newline at end of file
... ...