AgentImagesProcessor modifs.

Alain Klotz
1 parent a5160272
Showing 3 changed files with 214 additions and 40 deletions Show diff stats
config/schemas/schema_observatory-2.0.yml
privatedev/config/tnc/observatory_tnc.yml
src/core/pyros_django/observation_manager/AgentImagesProcessor.py
@@ -35,6 +35,7 @@ schema;schema_AGENT_DEVICE:
       type: str
     device: 
       type: str
+    # if protocol is not defined, the default value will be os.environ['PROJECT_ROOT_PATH']/src/core/pyros_django
     protocol: 
       type: str
     is_real:
@@ -266,10 +266,14 @@ OBSERVATORY:
                 # SF11-IPC (for AK)
                 - AGENT:
-                    name: AgentImagesProcessor_tnc_up1
+                    name: AgentImagesProcessor
                     computer: MainComputer
-                    protocol: private/plugin/agent/AgentImagesProcessor_tnc_up1.py
+                # SF11-IPC (for AK)
+                - AGENT:
+                    name: AgentImagesProcessor_tnc_up1
+                    computer: MainComputer
+                    protocol: privatedev/plugin/agent/AgentImagesProcessor_tnc_up1.py
             TOPOLOGY:
 #!/usr/bin/env python3
+#
+# To launch this agent from the root of Pyros:
+# cd /srv/develop/pyros
+# .\PYROS -t start agentImagesProcessor -o tnc -fg
+#
+# ---------------------------------------------------
 import sys
 ##import utils.Logger as L
@@ -18,24 +24,23 @@ from common.models import Sequence
 ##log = L.setupLogger("AgentXTaskLogger", "AgentX")
-
+# = aklotz =====================================================
+import os
+import glob
+import shutil
+import guitastro
+# --- How to avoid the next append ?
+path = "/home/pyros_user/app/vendor/guitastro/"
+if path not in sys.path:
+    sys.path.append(path)
+# = aklotz =====================================================
 class AgentImagesProcessor(Agent):
-
-    # FOR TEST ONLY
-    # Run this agent in simulator mode
-    TEST_MODE = False
-    # Run the assertion tests at the end
-    TEST_WITH_FINAL_TEST = True
-    TEST_MAX_DURATION_SEC = None
-    #TEST_MAX_DURATION_SEC = 120
-
-    # PM 20190416 fucking config path starting: /home/patrick/Dev/PYROS/start_agent.py agentM
-    ##_path_data = 'config'
-    _path_data = 'config/old_config'
-
-    log.debug("PLC instanciated")
+    # - All possible running states
+    RUNNING_NOTHING = 0
+    RUNNING_ONE_IMAGE_PROCESSING = 1
+    RUNNING_COMPUTE_RON_GAIN = 2
     AGENT_SPECIFIC_COMMANDS = [
         #"do_replan",
@@ -63,27 +68,81 @@ class AgentImagesProcessor(Agent):
         FUNCTIONS RUN INSIDE MAIN THREAD
     =================================================================
     """
-    # old config
-    # @override
-    #def __init__(self, name:str=None, config_filename=None, RUN_IN_THREAD=True):
-    # def __init__(self, config_filename=None, RUN_IN_THREAD=True):
-    #     ##if name is None: name = self.__class__.__name__
-    #     super().__init__(config_filename, RUN_IN_THREAD)
-    # new config (obsconfig)
-    #def __init__(self, name:str=None, RUN_IN_THREAD=True):
     def __init__(self, name:str=None):
         if name is None:
             name = self.__class__.__name__
         super().__init__()
-        #super().__init__(RUN_IN_THREAD)
+        
     # @override
     def init(self):
         super().init()
         log.debug("end init()")
-        # --- Set the mode according the startmode value
-        ##agent_alias = self.__class__.__name__
-        ##self.set_mode_from_config(agent_alias)
+
+        # = aklotz =====================================================
+
+        #self.TEST_MODE = True
+        log.info(f"self.TEST_MODE = {self.TEST_MODE}")
+        
+        # === Get config infos
+        # in particular, get self._path_data_root where data to process are.
+        agent_alias = self.__class__.__name__
+        log.info(f"agent_alias = {agent_alias}")
+        config = self._oc['config']
+        units = config.get_units()
+        kunits = list(units.keys())
+        kunit = kunits[0]
+        log.info(f"unit key = {kunit}")
+        unit = units[kunit]
+        self._unit = unit
+        # log.info(f"unit = {unit}")
+        agents = config.get_agents(kunit)
+        # log.info(f"agents = {agents}")        
+        attrs = agents[agent_alias]
+        key = 'path_data_root'
+        if key in attrs:
+            self._path_data_root = attrs['path_data_root']
+        else:
+            # default with docker should be /home/pyros_user/app
+            self._path_data_root = os.environ['PROJECT_ROOT_PATH']
+        if self._path_data_root == None:
+            self._path_data_root = os.environ['PROJECT_ROOT_PATH']
+        log.info(f"path_data_root = {self._path_data_root}")
+
+        # === Define and create the directories for the images to process
+        # place this paragraph in a common class for camera device and image processing
+        self._paths = {}
+        self._paths['ima_incoming'] = os.path.join(self._path_data_root,"data/images/incoming")
+        self._paths['ima_processed'] = os.path.join(self._path_data_root,"data/images/processed")
+        self._paths['ima_tmp'] = os.path.join(self._path_data_root,"data/images/tmp")
+        self._paths['ima_darks'] = os.path.join(self._path_data_root,"data/images/darks")
+        self._paths['ima_flats'] = os.path.join(self._path_data_root,"data/images/flats")
+        self._paths['ima_bias'] = os.path.join(self._path_data_root,"data/images/bias")
+        for path in self._paths.values():
+            os.makedirs(path, exist_ok = True)
+
+        # === Define an image to test the processing and copy it in incoming directory
+        if self.TEST_MODE == True:
+            self._file_ima_test = os.path.join(self._path_data_root,"vendor/guitastro/test/data/m57.fit")
+            file_in = self._file_ima_test
+            file_out = f"{self._paths['ima_incoming']}/m57.fit"
+            shutil.copyfile(file_in, file_out)
+        
+        # === Instanciate an object Ima to make image processing
+        self._ima = guitastro.Ima()
+        home = guitastro.Home(self._unit['home'])
+        
+        # === Get longitude of the unit to generate the night yyyymmdd and subdirectories yyyy/mm/dd
+        longitude = home.longitude
+        log.info(f"Longitude={longitude}")
+        self._ima.longitude(longitude)
+        log.info("Init done with success")
+        
+        # === Status of routine processing
+        self._routine_running = self.RUNNING_NOTHING
+        
+        # = aklotz =====================================================
+    
     '''
     # @override
@@ -107,30 +166,140 @@ class AgentImagesProcessor(Agent):
     def do_log(self):
         super().do_log()
-    def apply_calibration_files(self):
-        print("\n Starting calibration of new image files...\n")
-        time.sleep(5)
+    def glob_images_to_process(self):
+    
+        # = aklotz =====================================================
+        # - glob the incoming directory:
+        fitsfiles = glob.glob(f"{self._paths['ima_incoming']}/*.fit")
+        # - Please sort list of files in increasing dates (TODO)
+        return fitsfiles
+
+    def bias_correction(self):
+
+        # - Search the bias
+        path_bias = os.path.join( self._paths['ima_bias'], self._date_night )
+        fitsbiasfiles = glob.glob(f"{path_bias}/*.fit")
+        log.info(f"fitsbiasfiles = {fitsbiasfiles}")
+        if len(fitsbiasfiles) > 0:
+        
+            # - Select the bias
+            pass
+
+    def dark_correction(self):
+
+        # - Search the dark
+        path_darks = os.path.join( self._paths['ima_darks'], self._date_night )
+        fitsdarkfiles = glob.glob(f"{path_darks}/*.fit")
+        log.info(f"fitsdarkfiles = {fitsdarkfiles}")        
+        if len(fitsdarkfiles) > 0:
+        
+            # - Select two darks and compute the therm using exposure
+            # - Correction of dark
+            pass
-        # TODO:
+    def flat_correction(self):
+        # - Search the flat
+        path_flats = os.path.join( self._paths['ima_flats'], self._date_night )
+        fitsflatfiles = glob.glob(f"{path_flats}/*.fit")
+        log.info(f"fitsflatfiles = {fitsflatfiles}")
+        if len(fitsflatfiles) > 0:
+            # - Select the flat (with the filter)
+            # - Correction of flat
+            pass
+    
+    def process_one_image(self, fitsfile: str):
+        
+        # = aklotz =====================================================
+
+        # - Load file in memory
+        log.info("Load the file in memory")
+        f = self._ima.genename(self._ima.load(fitsfile))
+        # log.info(f"f={f}")
+
+        # - Save as tmp
+        self._ima.path(self._paths['ima_tmp'])
+        log.info("Save the temporary file as tmp name")   
+        self._ima.save("tmp")
+
+        # - Load tmp and get infos
+        self._ima.load("tmp")
+        date_obs = self._ima.getkwd("DATE-OBS")
+        self._date_night = self._ima.get_night(date_obs)
+        log.info(f"Date_obs = {date_obs}")
+        log.info(f"Night = {self._date_night}")
+        exposure = self._ima.getkwd("EXPOSURE")
+        log.info(f"Exposure = {exposure}")
+
+        # - Bias correction
+        self.bias_correction()
+
+        # - Dark correction
+        self.dark_correction()
+
+        # - Flat correction
+        self.flat_correction()
+
+        # - Save tmp corrected by dark and flat
+        self._ima.path(self._paths['ima_tmp'])
+        self._ima.save("tmp")
+
+        # - Inversion of mirrors or mirorxy (TODO)
+        
+        # - Cosmetic correction (TODO)
+        
+        # - WCS calibration (TODO)
+        
+        # - Prepare the output file name
+        log.info("Decode the filename")
+        fgen_in = f['genename'] + f['sep'] + f['indexes'][0] + f['suffix']
+        fext_in = f['file_extension']            
+        fext_out = ".fits"
+
+        # - Save in processed
+        yyyy = self._date_night[0:4]
+        mm = self._date_night[4:6]
+        dd = self._date_night[6:8]
+        path_processed = os.path.join( self._paths['ima_processed'], yyyy, mm, dd )
+        self._ima.path(path_processed)
+        fname_out = fgen_in + fext_out
+        fname = self._ima.save(fname_out)
+        log.info(f"Save the processed image {fname}")
+
+        # - Delete the file in incoming directory
+        os.remove(fitsfile)
+        log.info(f"Delete the raw image {fitsfile}")
+
+        # - Update the running state
+        self._routine_running = self.RUNNING_NOTHING
+        
+        # ---
+        # = aklotz =====================================================
+
+        time.sleep(5)
         print("\n ...End of images calibration\n")
     # Note : called by _routine_process() in Agent
     # @override
     def routine_process_before_body(self):
-        print("The Observatory configuration :")
-        self.show_config()
         log.debug("in routine_process_before_body()")
-        #self.apply_calibration_files()
     # Note : called by _routine_process() in Agent
     # @override
     def routine_process_after_body(self):
-        print("The Observatory configuration :")
-        #self.show_config()
         log.debug("in routine_process_after_body()")
-        self.apply_calibration_files()
-
+        if self._routine_running == self.RUNNING_NOTHING:
+            # Get files to process
+            fitsfiles = self.glob_images_to_process()
+            n = len(fitsfiles)
+            log.info(f"There are {n} images to process")
+            if n > 0:
+                # - We select the oldest image
+                fitsfile = fitsfiles[0]
+                log.info(f"Process the file {fitsfile}")
+                # - Thread TODO
+                self._routine_running = self.RUNNING_ONE_IMAGE_PROCESSING
+                self.process_one_image(fitsfile)
     '''
     # @override