nc2cdf.py
7.33 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
#!/usr/bin/env python
# -*- coding: utf-8 -*
# Resources:
# - http://pythonhosted.org/SpacePy/pycdf.html
# - http://unidata.github.io/netcdf4-python/
import os
import os.path as op
from datetime import datetime
os.environ['SPACEPY'] = '/tmp'
import netCDF4
from spacepy import pycdf
TEMP_FILE = '/tmp'
RESOLVER_URL = 'http://apus.irap.omp.eu:8080/amda-registry/resolver'
def error(error_message, e=None):
""" Display an `error_message` and exit the program. """
import sys
with open(op.join(TEMP_FILE, 'converter.log'), 'a') as f_log:
f_log.write('With arguments ' + ', '.join(sys.argv))
f_log.write(error_message)
sys.stderr.write(error_message + ((':\n %s\n' % e) if e else '\n'))
sys.exit(1)
class NetCdf:
SPASE_DATE_FORMAT = '%Y%j%H%M%S' # ex: 2016238000000
def __init__(self, input_path):
"""Constructor. Load a NetCDF file from `self.netcdf_path`, return a NetCDF4 object."""
self.cdf_path = None
self.cdf = None
self.temp_nc = None
extension = input_path.split('/')[-1].split('.', 1)[1]
if extension == 'nc.gz':
self.unzip(input_path)
elif extension == 'nc':
self.netcdf_path = input_path
else:
error('Unknown file extension "%s".' % extension)
try:
self.netcdf = netCDF4.Dataset(self.netcdf_path)
except Exception as e:
error('Can not open netCDF file %s' % self.netcdf_path, e)
if self.temp_nc:
os.remove(self.temp_nc)
def unzip(self, ncgz_input_path):
import gzip
from shutil import copyfileobj
netcdf_temp_path = self.build_temp_path(ncgz_input_path, '.nc')
if not op.exists(ncgz_input_path):
error('Compressed Net-CDF file is not found in "%s".' % ncgz_input_path)
try:
with gzip.open(ncgz_input_path, 'rb') as f_in, open(netcdf_temp_path, 'wb') as f_out:
copyfileobj(f_in, f_out)
except Exception as e:
error('Can not unzip compressed Net-CDF from %s to %s' % (ncgz_input_path, netcdf_temp_path), e)
self.temp_nc = netcdf_temp_path
self.netcdf_path = netcdf_temp_path
def parse_date(self, str_date):
""" Parse the string `str_date` and return the date."""
return datetime.strptime(str_date[:4] + str(int(str_date[4:7]) + 1) + str_date[7:13], self.SPASE_DATE_FORMAT)
# NetCDF methods
def get_netcdf_path(self):
"""Return the NetCDF path."""
return self.netcdf_path
def get_netcdf(self):
"""Return the NetCDF object."""
return self.netcdf
def describe(self):
"""Display all NetCDF variables of the NetCF file"""
def describe_var(var):
"""Describe an net-cdf variable."""
print('== %s ==' % var.name)
print(' - numpy type: %s ^ %s ' % (str(var.dtype), str(var.ndim)))
print(' - dimension(s): %s' % ', '.join(list(var.dimensions)))
print(' - size: %s = %d' % ('x'.join([str(n) for n in var.shape]), var.size))
if var.ndim == 1 and str(var.dtype) == '|S1':
print(' - values: \'%s\', ...' % ''.join([c.decode('utf-8') for c in var[:]]))
if var.ncattrs():
print(' - Attributes:')
for var_attr_name in var.ncattrs():
print(' - %s: %s' % (var_attr_name, getattr(var, var_attr_name)))
for (key, value) in self.netcdf.variables.items():
describe_var(value)
print('== Global attributes ==')
for global_attr_name in self.netcdf.ncattrs():
print(' - %s: %s' % (global_attr_name, getattr(self.netcdf, global_attr_name)))
# CDF methods
@staticmethod
def create_new_cdf(cdf_path):
"""
Create a new empty CDF file in `self.cdf_path`, return a pyCDF object.
"""
# Create and clean a new directory for the CDF file.
cdf_dir = op.dirname(cdf_path)
if not op.exists(cdf_dir):
try:
os.makedirs(cdf_dir)
except IOError as e:
error('Can not create directory %s' % cdf_dir, e)
if op.exists(cdf_path):
try:
os.remove(cdf_path)
except IOError as e:
error('A CDF file already exist in %s and it can not be removed' % cdf_path, e)
# Make the pyCDF object
try:
cdf = pycdf.CDF(cdf_path, '')
except pycdf.CDFError as e:
error('Can not create CDF file on %s, check that the directory exists and its writing access' % cdf_path, e)
# TODO Get spase file
get_spase()
cdf.attrs['Acknowledgement '] = 'John Doe'
return cdf
def get_spase(self):
@staticmethod
def build_temp_path(path, extension):
return op.abspath(op.join(TEMP_FILE, op.basename(path).split('.')[0] + extension))
def get_cdf(self, cdf_path=None):
"""
Convert and return the CDF object (only return it if it's already converted)
- `cdf_path`: The path of the CDF file (needed for CDF creation), or a temp path if not specified.
"""
if self.cdf:
return self.cdf
self.cdf_path = op.abspath(cdf_path) if cdf_path else self.build_temp_path(self.netcdf_path, '.cdf')
self.cdf = self.create_new_cdf(self.cdf_path)
for global_attr_name in self.netcdf.ncattrs():
self.cdf.attrs[global_attr_name] = getattr(self.netcdf, global_attr_name)
for key, var in self.netcdf.variables.items():
dimensions = list(var.dimensions)
if str(var.dtype) == '|S1':
if len(dimensions) == 1:
var_str = str(netCDF4.chartostring(var[:]).astype(str, copy=False))
self.cdf.attrs[key] = self.parse_date(var_str) if dimensions[0] == 'TimeLength' else var_str
elif len(dimensions) == 2:
var_strs = map(str, netCDF4.chartostring(var[:]).astype(str, copy=False))
self.cdf[key] = list(map(self.parse_date, var_strs) if dimensions[1] == 'TimeLength' else var_strs)
else:
self.cdf[key] = var[:]
for var_attr_name in var.ncattrs():
self.cdf[key].attrs[var_attr_name] = getattr(var, var_attr_name)
return self.cdf
def get_cdf_path(self):
return self.cdf_path
if __name__ == '__main__':
import sys
if len(sys.argv) not in (2, 3):
print('usage:')
print('- `%s path/to/file.nc.gz` # Convert a Net-CDF file, '
'save it in a temp directory, then display its path.' % sys.argv[0])
print('- `%s -i path/to/file.nc.gz` # Display information about a Net-CDF file.' % sys.argv[0])
print('- `%s path/to/file.nc.gz path/to/file.cdf` # Convert a Net-CDF file.'
'and save it in the specified path.' % sys.argv[0])
print('This script can also be used as a Python library.')
exit(1)
if len(sys.argv) == 2:
netcdf = NetCdf(sys.argv[1])
netcdf.get_cdf()
print('File stored in "%s".' % netcdf.get_cdf_path())
elif len(sys.argv) == 3:
if sys.argv[1] == '-i':
netcdf = NetCdf(sys.argv[2])
netcdf.describe()
else:
netcdf = NetCdf(sys.argv[1])
netcdf.get_cdf(sys.argv[2])