From e6a63db3001bc4b5cd7f2b7b22d7aaabc51430de Mon Sep 17 00:00:00 2001 From: Alain Klotz Date: Wed, 22 Jun 2022 17:05:14 +0200 Subject: [PATCH] AgentImagesProcessor modifs. --- config/schemas/schema_observatory-2.0.yml | 1 + privatedev/config/tnc/observatory_tnc.yml | 8 ++++++-- src/core/pyros_django/observation_manager/AgentImagesProcessor.py | 245 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------------------------- 3 files changed, 214 insertions(+), 40 deletions(-) diff --git a/config/schemas/schema_observatory-2.0.yml b/config/schemas/schema_observatory-2.0.yml index ceef805..5f2f6e3 100644 --- a/config/schemas/schema_observatory-2.0.yml +++ b/config/schemas/schema_observatory-2.0.yml @@ -35,6 +35,7 @@ schema;schema_AGENT_DEVICE: type: str device: type: str + # if protocol is not defined, the default value will be os.environ['PROJECT_ROOT_PATH']/src/core/pyros_django protocol: type: str is_real: diff --git a/privatedev/config/tnc/observatory_tnc.yml b/privatedev/config/tnc/observatory_tnc.yml index 802988c..1ba6895 100644 --- a/privatedev/config/tnc/observatory_tnc.yml +++ b/privatedev/config/tnc/observatory_tnc.yml @@ -266,10 +266,14 @@ OBSERVATORY: # SF11-IPC (for AK) - AGENT: - name: AgentImagesProcessor_tnc_up1 + name: AgentImagesProcessor computer: MainComputer - protocol: private/plugin/agent/AgentImagesProcessor_tnc_up1.py + # SF11-IPC (for AK) + - AGENT: + name: AgentImagesProcessor_tnc_up1 + computer: MainComputer + protocol: privatedev/plugin/agent/AgentImagesProcessor_tnc_up1.py TOPOLOGY: diff --git a/src/core/pyros_django/observation_manager/AgentImagesProcessor.py b/src/core/pyros_django/observation_manager/AgentImagesProcessor.py index db7da7d..79bae1e 100755 --- a/src/core/pyros_django/observation_manager/AgentImagesProcessor.py +++ b/src/core/pyros_django/observation_manager/AgentImagesProcessor.py @@ -1,4 +1,10 @@ #!/usr/bin/env python3 +# +# To launch this agent from the root of Pyros: +# cd /srv/develop/pyros +# .\PYROS -t start agentImagesProcessor -o tnc -fg +# +# --------------------------------------------------- import sys ##import utils.Logger as L @@ -18,24 +24,23 @@ from common.models import Sequence ##log = L.setupLogger("AgentXTaskLogger", "AgentX") - +# = aklotz ===================================================== +import os +import glob +import shutil +import guitastro +# --- How to avoid the next append ? +path = "/home/pyros_user/app/vendor/guitastro/" +if path not in sys.path: + sys.path.append(path) +# = aklotz ===================================================== class AgentImagesProcessor(Agent): - - # FOR TEST ONLY - # Run this agent in simulator mode - TEST_MODE = False - # Run the assertion tests at the end - TEST_WITH_FINAL_TEST = True - TEST_MAX_DURATION_SEC = None - #TEST_MAX_DURATION_SEC = 120 - - # PM 20190416 fucking config path starting: /home/patrick/Dev/PYROS/start_agent.py agentM - ##_path_data = 'config' - _path_data = 'config/old_config' - - log.debug("PLC instanciated") + # - All possible running states + RUNNING_NOTHING = 0 + RUNNING_ONE_IMAGE_PROCESSING = 1 + RUNNING_COMPUTE_RON_GAIN = 2 AGENT_SPECIFIC_COMMANDS = [ #"do_replan", @@ -63,27 +68,81 @@ class AgentImagesProcessor(Agent): FUNCTIONS RUN INSIDE MAIN THREAD ================================================================= """ - # old config - # @override - #def __init__(self, name:str=None, config_filename=None, RUN_IN_THREAD=True): - # def __init__(self, config_filename=None, RUN_IN_THREAD=True): - # ##if name is None: name = self.__class__.__name__ - # super().__init__(config_filename, RUN_IN_THREAD) - # new config (obsconfig) - #def __init__(self, name:str=None, RUN_IN_THREAD=True): def __init__(self, name:str=None): if name is None: name = self.__class__.__name__ super().__init__() - #super().__init__(RUN_IN_THREAD) + # @override def init(self): super().init() log.debug("end init()") - # --- Set the mode according the startmode value - ##agent_alias = self.__class__.__name__ - ##self.set_mode_from_config(agent_alias) + + # = aklotz ===================================================== + + #self.TEST_MODE = True + log.info(f"self.TEST_MODE = {self.TEST_MODE}") + + # === Get config infos + # in particular, get self._path_data_root where data to process are. + agent_alias = self.__class__.__name__ + log.info(f"agent_alias = {agent_alias}") + config = self._oc['config'] + units = config.get_units() + kunits = list(units.keys()) + kunit = kunits[0] + log.info(f"unit key = {kunit}") + unit = units[kunit] + self._unit = unit + # log.info(f"unit = {unit}") + agents = config.get_agents(kunit) + # log.info(f"agents = {agents}") + attrs = agents[agent_alias] + key = 'path_data_root' + if key in attrs: + self._path_data_root = attrs['path_data_root'] + else: + # default with docker should be /home/pyros_user/app + self._path_data_root = os.environ['PROJECT_ROOT_PATH'] + if self._path_data_root == None: + self._path_data_root = os.environ['PROJECT_ROOT_PATH'] + log.info(f"path_data_root = {self._path_data_root}") + + # === Define and create the directories for the images to process + # place this paragraph in a common class for camera device and image processing + self._paths = {} + self._paths['ima_incoming'] = os.path.join(self._path_data_root,"data/images/incoming") + self._paths['ima_processed'] = os.path.join(self._path_data_root,"data/images/processed") + self._paths['ima_tmp'] = os.path.join(self._path_data_root,"data/images/tmp") + self._paths['ima_darks'] = os.path.join(self._path_data_root,"data/images/darks") + self._paths['ima_flats'] = os.path.join(self._path_data_root,"data/images/flats") + self._paths['ima_bias'] = os.path.join(self._path_data_root,"data/images/bias") + for path in self._paths.values(): + os.makedirs(path, exist_ok = True) + + # === Define an image to test the processing and copy it in incoming directory + if self.TEST_MODE == True: + self._file_ima_test = os.path.join(self._path_data_root,"vendor/guitastro/test/data/m57.fit") + file_in = self._file_ima_test + file_out = f"{self._paths['ima_incoming']}/m57.fit" + shutil.copyfile(file_in, file_out) + + # === Instanciate an object Ima to make image processing + self._ima = guitastro.Ima() + home = guitastro.Home(self._unit['home']) + + # === Get longitude of the unit to generate the night yyyymmdd and subdirectories yyyy/mm/dd + longitude = home.longitude + log.info(f"Longitude={longitude}") + self._ima.longitude(longitude) + log.info("Init done with success") + + # === Status of routine processing + self._routine_running = self.RUNNING_NOTHING + + # = aklotz ===================================================== + ''' # @override @@ -107,30 +166,140 @@ class AgentImagesProcessor(Agent): def do_log(self): super().do_log() - def apply_calibration_files(self): - print("\n Starting calibration of new image files...\n") - time.sleep(5) + def glob_images_to_process(self): + + # = aklotz ===================================================== + # - glob the incoming directory: + fitsfiles = glob.glob(f"{self._paths['ima_incoming']}/*.fit") + # - Please sort list of files in increasing dates (TODO) + return fitsfiles + + def bias_correction(self): + + # - Search the bias + path_bias = os.path.join( self._paths['ima_bias'], self._date_night ) + fitsbiasfiles = glob.glob(f"{path_bias}/*.fit") + log.info(f"fitsbiasfiles = {fitsbiasfiles}") + if len(fitsbiasfiles) > 0: + + # - Select the bias + pass + + def dark_correction(self): + + # - Search the dark + path_darks = os.path.join( self._paths['ima_darks'], self._date_night ) + fitsdarkfiles = glob.glob(f"{path_darks}/*.fit") + log.info(f"fitsdarkfiles = {fitsdarkfiles}") + if len(fitsdarkfiles) > 0: + + # - Select two darks and compute the therm using exposure + # - Correction of dark + pass - # TODO: + def flat_correction(self): + # - Search the flat + path_flats = os.path.join( self._paths['ima_flats'], self._date_night ) + fitsflatfiles = glob.glob(f"{path_flats}/*.fit") + log.info(f"fitsflatfiles = {fitsflatfiles}") + if len(fitsflatfiles) > 0: + # - Select the flat (with the filter) + # - Correction of flat + pass + + def process_one_image(self, fitsfile: str): + + # = aklotz ===================================================== + + # - Load file in memory + log.info("Load the file in memory") + f = self._ima.genename(self._ima.load(fitsfile)) + # log.info(f"f={f}") + + # - Save as tmp + self._ima.path(self._paths['ima_tmp']) + log.info("Save the temporary file as tmp name") + self._ima.save("tmp") + + # - Load tmp and get infos + self._ima.load("tmp") + date_obs = self._ima.getkwd("DATE-OBS") + self._date_night = self._ima.get_night(date_obs) + log.info(f"Date_obs = {date_obs}") + log.info(f"Night = {self._date_night}") + exposure = self._ima.getkwd("EXPOSURE") + log.info(f"Exposure = {exposure}") + + # - Bias correction + self.bias_correction() + + # - Dark correction + self.dark_correction() + + # - Flat correction + self.flat_correction() + + # - Save tmp corrected by dark and flat + self._ima.path(self._paths['ima_tmp']) + self._ima.save("tmp") + + # - Inversion of mirrors or mirorxy (TODO) + + # - Cosmetic correction (TODO) + + # - WCS calibration (TODO) + + # - Prepare the output file name + log.info("Decode the filename") + fgen_in = f['genename'] + f['sep'] + f['indexes'][0] + f['suffix'] + fext_in = f['file_extension'] + fext_out = ".fits" + + # - Save in processed + yyyy = self._date_night[0:4] + mm = self._date_night[4:6] + dd = self._date_night[6:8] + path_processed = os.path.join( self._paths['ima_processed'], yyyy, mm, dd ) + self._ima.path(path_processed) + fname_out = fgen_in + fext_out + fname = self._ima.save(fname_out) + log.info(f"Save the processed image {fname}") + + # - Delete the file in incoming directory + os.remove(fitsfile) + log.info(f"Delete the raw image {fitsfile}") + + # - Update the running state + self._routine_running = self.RUNNING_NOTHING + + # --- + # = aklotz ===================================================== + + time.sleep(5) print("\n ...End of images calibration\n") # Note : called by _routine_process() in Agent # @override def routine_process_before_body(self): - print("The Observatory configuration :") - self.show_config() log.debug("in routine_process_before_body()") - #self.apply_calibration_files() # Note : called by _routine_process() in Agent # @override def routine_process_after_body(self): - print("The Observatory configuration :") - #self.show_config() log.debug("in routine_process_after_body()") - self.apply_calibration_files() - + if self._routine_running == self.RUNNING_NOTHING: + # Get files to process + fitsfiles = self.glob_images_to_process() + n = len(fitsfiles) + log.info(f"There are {n} images to process") + if n > 0: + # - We select the oldest image + fitsfile = fitsfiles[0] + log.info(f"Process the file {fitsfile}") + # - Thread TODO + self._routine_running = self.RUNNING_ONE_IMAGE_PROCESSING + self.process_one_image(fitsfile) ''' # @override -- libgit2 0.21.2