First commit

Nathanael Jourdane
Showing 6 changed files with 364 additions and 0 deletions Show diff stats
conda_env_nc2cdf.yml
examples/mag_1981244213334741.nc.gz
examples/skr151150000.nc.gz
nc2cdf.py
pip_req_nc2cdf.txt
readme.md
+++ a/conda_env_nc2cdf.yml
@@ -0,0 +1,38 @@
+name: nc2cdf
+channels:
+- defaults
+- http://conda.binstar.org/roipoussiere
+dependencies:
+- curl=7.49.0=1
+- hdf4=4.2.11=0
+- hdf5=1.8.17=1
+- jpeg=9b=0
+- libnetcdf=4.4.1=0
+- mkl=2017.0.1=0
+- netcdf4=1.2.4=np111py36_0
+- numpy=1.11.3=py36_0
+- openssl=1.0.2k=0
+- pip=9.0.1=py36_1
+- python=3.6.0=0
+- readline=6.2=2
+- setuptools=27.2.0=py36_0
+- sqlite=3.13.0=0
+- tk=8.5.18=0
+- wheel=0.29.0=py36_0
+- xz=5.2.2=1
+- zlib=1.2.8=3
+- pip:
+  - cycler==0.10.0
+  - decorator==4.0.11
+  - ffnet==0.8.3
+  - h5py==2.6.0
+  - matplotlib==2.0.0
+  - networkx==1.11
+  - pyparsing==2.1.10
+  - python-dateutil==2.6.0
+  - pytz==2016.10
+  - scipy==0.18.1
+  - six==1.10.0
+  - spacepy==0.1.6
+prefix: /home/nathanael/.anaconda3/envs/nc2cdf
+
+++ a/nc2cdf.py
@@ -0,0 +1,210 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*
+
+# Resources:
+# - http://pythonhosted.org/SpacePy/pycdf.html
+# - http://unidata.github.io/netcdf4-python/
+
+import os
+import os.path as op
+from datetime import datetime
+
+os.environ['SPACEPY'] = '/tmp'
+
+import netCDF4
+from spacepy import pycdf
+
+AMDA_DICT_PATH = './amda.json'
+TEMP_FILE = '/tmp'
+
+
+def error(error_message, e=None):
+    """ Display an `error_message` and exit the program. """
+
+    import sys
+
+    with open(op.join(TEMP_FILE, 'converter.log'), 'a') as f_log:
+        f_log.write('With arguments ' + ', '.join(sys.argv))
+        f_log.write(error_message)
+
+    sys.stderr.write(error_message + ((':\n  %s\n' % e) if e else '\n'))
+    sys.exit(1)
+
+
+class NetCdf:
+    SPASE_DATE_FORMAT = '%Y%j%H%M%S'  # ex: 2016238000000
+
+    def __init__(self, input_path):
+        """Constructor. Load a NetCDF file from `self.netcdf_path`, return a NetCDF4 object."""
+
+        self.cdf_path = None
+        self.cdf = None
+        self.temp_nc = None
+
+        extension = input_path.split('/')[-1].split('.', 1)[1]
+        if extension == 'nc.gz':
+            self.unzip(input_path)
+        elif extension == 'nc':
+            self.netcdf_path = input_path
+        else:
+            error('Unknown file extension "%s".' % extension)
+
+        try:
+            self.netcdf = netCDF4.Dataset(self.netcdf_path)
+        except Exception as e:
+            error('Can not open netCDF file %s' % self.netcdf_path, e)
+
+        if self.temp_nc:
+            os.remove(self.temp_nc)
+
+    def unzip(self, ncgz_input_path):
+        import gzip
+        from shutil import copyfileobj
+
+        netcdf_temp_path = self.build_temp_path(ncgz_input_path, '.nc')
+
+        if not op.exists(ncgz_input_path):
+            error('Compressed Net-CDF file is not found in "%s".' % ncgz_input_path)
+
+        try:
+            with gzip.open(ncgz_input_path, 'rb') as f_in, open(netcdf_temp_path, 'wb') as f_out:
+                copyfileobj(f_in, f_out)
+        except Exception as e:
+            error('Can not unzip compressed Net-CDF from %s to %s' % (ncgz_input_path, netcdf_temp_path), e)
+
+        self.temp_nc = netcdf_temp_path
+        self.netcdf_path = netcdf_temp_path
+
+    def parse_date(self, str_date):
+        """ Parse the string `str_date` and return the date."""
+
+        return datetime.strptime(str_date[:4] + str(int(str_date[4:7]) + 1) + str_date[7:13], self.SPASE_DATE_FORMAT)
+
+    # NetCDF methods
+
+    def get_netcdf_path(self):
+        """Return the NetCDF path."""
+
+        return self.netcdf_path
+
+    def get_netcdf(self):
+        """Return the NetCDF object."""
+
+        return self.netcdf
+
+    def describe(self):
+        """Display all NetCDF variables of the NetCF file"""
+
+        def describe_var(var):
+            """Describe an net-cdf variable."""
+
+            print('== %s ==' % var.name)
+            print('  - numpy type: %s ^ %s ' % (str(var.dtype), str(var.ndim)))
+            print('  - dimension(s): %s' % ', '.join(list(var.dimensions)))
+            print('  - size: %s = %d' % ('x'.join([str(n) for n in var.shape]), var.size))
+            if var.ndim == 1 and str(var.dtype) == '|S1':
+                print('  - values: \'%s\', ...' % ''.join([c.decode('utf-8') for c in var[:]]))
+
+            if var.ncattrs():
+                print('  - Attributes:')
+                for var_attr_name in var.ncattrs():
+                    print('      - %s: %s' % (var_attr_name, getattr(var, var_attr_name)))
+
+        for (key, value) in self.netcdf.variables.items():
+            describe_var(value)
+
+        print('== Global attributes ==')
+        for global_attr_name in self.netcdf.ncattrs():
+            print('  - %s: %s' % (global_attr_name, getattr(self.netcdf, global_attr_name)))
+
+    # CDF methods
+
+    @staticmethod
+    def create_new_cdf(cdf_path):
+        """
+        Create a new empty CDF file in `self.cdf_path`, return a pyCDF object.
+        """
+
+        # Create and clean a new directory for the CDF file.
+        cdf_dir = op.dirname(cdf_path)
+        if not op.exists(cdf_dir):
+            try:
+                os.makedirs(cdf_dir)
+            except IOError as e:
+                error('Can not create directory %s' % cdf_dir, e)
+        if op.exists(cdf_path):
+            try:
+                os.remove(cdf_path)
+            except IOError as e:
+                error('A CDF file already exist in %s and it can not be removed' % cdf_path, e)
+
+        # Make the pyCDF object
+        try:
+            return pycdf.CDF(cdf_path, '')
+        except pycdf.CDFError as e:
+            error('Can not create CDF file on %s, check that the directory exists and its writing access' % cdf_path, e)
+
+    @staticmethod
+    def build_temp_path(path, extension):
+        return op.abspath(op.join(TEMP_FILE, op.basename(path).split('.')[0] + extension))
+
+    def get_cdf(self, cdf_path=None):
+        """
+        Convert and return the CDF object (only return it if it's already converted)
+        - `cdf_path`: The path of the CDF file (needed for CDF creation), or a temp path if not specified.
+        """
+
+        if self.cdf:
+            return self.cdf
+
+        self.cdf_path = op.abspath(cdf_path) if cdf_path else self.build_temp_path(self.netcdf_path, '.cdf')
+        self.cdf = self.create_new_cdf(self.cdf_path)
+
+        for global_attr_name in self.netcdf.ncattrs():
+            self.cdf.attrs[global_attr_name] = getattr(self.netcdf, global_attr_name)
+
+        for key, var in self.netcdf.variables.items():
+            dimensions = list(var.dimensions)
+
+            if str(var.dtype) == '|S1':
+                if len(dimensions) == 1:
+                    var_str = str(netCDF4.chartostring(var[:]).astype(str, copy=False))
+                    self.cdf.attrs[key] = self.parse_date(var_str) if dimensions[0] == 'TimeLength' else var_str
+                elif len(dimensions) == 2:
+                    var_strs = map(str, netCDF4.chartostring(var[:]).astype(str, copy=False))
+                    self.cdf[key] = list(map(self.parse_date, var_strs) if dimensions[1] == 'TimeLength' else var_strs)
+            else:
+                self.cdf[key] = var[:]
+                for var_attr_name in var.ncattrs():
+                    self.cdf[key].attrs[var_attr_name] = getattr(var, var_attr_name)
+
+        return self.cdf
+
+    def get_cdf_path(self):
+        return self.cdf_path
+
+
+if __name__ == '__main__':
+    import sys
+
+    if len(sys.argv) not in (2, 3):
+        print('usage:')
+        print('- `%s path/to/file.nc.gz` # Convert a Net-CDF file, '
+              'save it in a temp directory, then display its path.' % sys.argv[0])
+        print('- `%s -i path/to/file.nc.gz` # Display information about a Net-CDF file.' % sys.argv[0])
+        print('- `%s path/to/file.nc.gz path/to/file.cdf` # Convert a Net-CDF file.'
+              'and save it in the specified path.' % sys.argv[0])
+        print('This script can also be used as a Python library.')
+        exit(1)
+
+    if len(sys.argv) == 2:
+        netcdf = NetCdf(sys.argv[1])
+        netcdf.get_cdf()
+        print('File stored in "%s".' % netcdf.get_cdf_path())
+    elif len(sys.argv) == 3:
+        if sys.argv[1] == '-i':
+            netcdf = NetCdf(sys.argv[2])
+            netcdf.describe()
+        else:
+            netcdf = NetCdf(sys.argv[1])
+            netcdf.get_cdf(sys.argv[2])
+++ a/pip_req_nc2cdf.txt
@@ -0,0 +1,16 @@
+appdirs==1.4.0
+cycler==0.10.0
+decorator==4.0.11
+functools32==3.2.3.post2
+h5py==2.6.0
+matplotlib==2.0.0
+netCDF4==1.2.7
+networkx==1.11
+numpy==1.12.0
+packaging==16.8
+pyparsing==2.1.10
+python-dateutil==2.6.0
+pytz==2016.10
+scipy==0.18.1
+six==1.10.0
+subprocess32==3.2.7
+++ a/readme.md
@@ -0,0 +1,100 @@
+# CDF tools
+
+## NetCDF to CDF converter
+
+- File: [nc2cdf.py](./nc2cdf.py)
+- Python interpreter: 3.6
+
+### CLI usage
+
+#### Converting a Net-CDF file:
+
+Convert the NetCDF file, save it in a temp directory, then display its path:
+
+    ./nc2cdf.py path/to/input_file.nc.gz
+
+Convert a Net-CDF file and save it in the specified path.
+
+    ./nc2cdf.py path/to/input_file.nc.gz path/to/output_file.cdf
+
+**Note:** If the specified input file is a gzip archive, it will be automatically extracted before the conversion.
+
+#### Describing a NetCDf file:
+
+    ./nc2cdf.py  -i path/to/file.nc.gz
+
+This display information about a Net-CDF file (such as global attributes and variables information).
+
+### Python library usage
+
+    import nc2cdf
+
+    netcdf = NetCdf('path/to/input_file.nc.gz')
+
+    netcdf.describe()
+
+    netcdf.get_cdf()
+    print('CDF path: ' + netcdf.get_cdf_path())
+
+	netcdf.get_cdf('path/to/output_file.cdf')
+
+## About NetCDF4 and pycdf
+
+### NetCDF4
+
+[NetCDF](https://www.unidata.ucar.edu/software/netcdf/) is C library to read and edit NetCDF files.
+
+[NetCDF4](https://github.com/Unidata/netcdf4-python) is a Python wrapper for NetCDF, which require the NetCDF library, used here to read NetCDF files.
+
+Documentation is available [here](http://unidata.github.io/netcdf4-python/).
+
+### pycdf
+
+[SpacePy](http://pythonhosted.org/SpacePy/index.html) is a python package for space sciences, used here to write CDF files.
+
+Documentation of the package spacepy.pycdf is available [here](http://pythonhosted.org/SpacePy/pycdf.htm).
+
+### Install the Python environment and dependencies
+
+We will install dependencies in Python environments.
+
+#### If you have NetCDF installed on your machine
+
+You can use pip and virtualenv:
+
+    pip install virtualenv
+    virtualenv -p python3 nc2cdf
+    source nc2cdf/bin/activate # Or ". nc2cdf/bin/activate.fish" on Fish terms
+    pip install -r pip_req_nc2cdf.txt
+
+#### If you don't have NetCDF installed on your machine
+
+The easier way is to use [Anaconda](https://docs.continuum.io/), which is a tool to install compiled Python dependencies in environments.
+
+1. [Install Anaconda3](https://docs.continuum.io/anaconda/install).
+2. Edit your system startup file:
+
+I recommend to add an alias which set the Python path. In this way the Anaconda Python will not be used by default and you can easily deal with multiple Anaconda versions.
+
+At the end of your `~/.bashrc`:
+
+    alias conda3="set PATH $HOME/.anaconda3/bin/ $PATH"
+
+Or on Fish terms (`~/.config/omf/init.fish`):
+
+    alias conda3="set PATH $HOME/.anaconda3/bin/ $PATH; and source $HOME/.anaconda3/etc/fish/conf.d/conda.fish
+
+3. Create the environment
+
+conda3
+conda create -f conda_env_nc2cdf.yml
+
+4. Activate the Conda environment:
+
+source activate nc2cdf # or only "activate nc2cdf" on Fish terms
+
+### Licence
+
+- License: [GPLv3](https://www.gnu.org/licenses/gpl-3.0.html);
+- Credits: CNRS/IRAP;
+- contact: nathanael.jourdane@irap.omp.eu