Commit e6a63db3001bc4b5cd7f2b7b22d7aaabc51430de

Authored by Alain Klotz
1 parent a5160272
Exists in dev

AgentImagesProcessor modifs.

config/schemas/schema_observatory-2.0.yml
... ... @@ -35,6 +35,7 @@ schema;schema_AGENT_DEVICE:
35 35 type: str
36 36 device:
37 37 type: str
  38 + # if protocol is not defined, the default value will be os.environ['PROJECT_ROOT_PATH']/src/core/pyros_django
38 39 protocol:
39 40 type: str
40 41 is_real:
... ...
privatedev/config/tnc/observatory_tnc.yml
... ... @@ -266,10 +266,14 @@ OBSERVATORY:
266 266  
267 267 # SF11-IPC (for AK)
268 268 - AGENT:
269   - name: AgentImagesProcessor_tnc_up1
  269 + name: AgentImagesProcessor
270 270 computer: MainComputer
271   - protocol: private/plugin/agent/AgentImagesProcessor_tnc_up1.py
272 271  
  272 + # SF11-IPC (for AK)
  273 + - AGENT:
  274 + name: AgentImagesProcessor_tnc_up1
  275 + computer: MainComputer
  276 + protocol: privatedev/plugin/agent/AgentImagesProcessor_tnc_up1.py
273 277  
274 278 TOPOLOGY:
275 279  
... ...
src/core/pyros_django/observation_manager/AgentImagesProcessor.py
1 1 #!/usr/bin/env python3
  2 +#
  3 +# To launch this agent from the root of Pyros:
  4 +# cd /srv/develop/pyros
  5 +# .\PYROS -t start agentImagesProcessor -o tnc -fg
  6 +#
  7 +# ---------------------------------------------------
2 8  
3 9 import sys
4 10 ##import utils.Logger as L
... ... @@ -18,24 +24,23 @@ from common.models import Sequence
18 24  
19 25 ##log = L.setupLogger("AgentXTaskLogger", "AgentX")
20 26  
21   -
  27 +# = aklotz =====================================================
  28 +import os
  29 +import glob
  30 +import shutil
  31 +import guitastro
  32 +# --- How to avoid the next append ?
  33 +path = "/home/pyros_user/app/vendor/guitastro/"
  34 +if path not in sys.path:
  35 + sys.path.append(path)
  36 +# = aklotz =====================================================
22 37  
23 38 class AgentImagesProcessor(Agent):
24 39  
25   -
26   - # FOR TEST ONLY
27   - # Run this agent in simulator mode
28   - TEST_MODE = False
29   - # Run the assertion tests at the end
30   - TEST_WITH_FINAL_TEST = True
31   - TEST_MAX_DURATION_SEC = None
32   - #TEST_MAX_DURATION_SEC = 120
33   -
34   - # PM 20190416 fucking config path starting: /home/patrick/Dev/PYROS/start_agent.py agentM
35   - ##_path_data = 'config'
36   - _path_data = 'config/old_config'
37   -
38   - log.debug("PLC instanciated")
  40 + # - All possible running states
  41 + RUNNING_NOTHING = 0
  42 + RUNNING_ONE_IMAGE_PROCESSING = 1
  43 + RUNNING_COMPUTE_RON_GAIN = 2
39 44  
40 45 AGENT_SPECIFIC_COMMANDS = [
41 46 #"do_replan",
... ... @@ -63,27 +68,81 @@ class AgentImagesProcessor(Agent):
63 68 FUNCTIONS RUN INSIDE MAIN THREAD
64 69 =================================================================
65 70 """
66   - # old config
67   - # @override
68   - #def __init__(self, name:str=None, config_filename=None, RUN_IN_THREAD=True):
69   - # def __init__(self, config_filename=None, RUN_IN_THREAD=True):
70   - # ##if name is None: name = self.__class__.__name__
71   - # super().__init__(config_filename, RUN_IN_THREAD)
72 71  
73   - # new config (obsconfig)
74   - #def __init__(self, name:str=None, RUN_IN_THREAD=True):
75 72 def __init__(self, name:str=None):
76 73 if name is None:
77 74 name = self.__class__.__name__
78 75 super().__init__()
79   - #super().__init__(RUN_IN_THREAD)
  76 +
80 77 # @override
81 78 def init(self):
82 79 super().init()
83 80 log.debug("end init()")
84   - # --- Set the mode according the startmode value
85   - ##agent_alias = self.__class__.__name__
86   - ##self.set_mode_from_config(agent_alias)
  81 +
  82 + # = aklotz =====================================================
  83 +
  84 + #self.TEST_MODE = True
  85 + log.info(f"self.TEST_MODE = {self.TEST_MODE}")
  86 +
  87 + # === Get config infos
  88 + # in particular, get self._path_data_root where data to process are.
  89 + agent_alias = self.__class__.__name__
  90 + log.info(f"agent_alias = {agent_alias}")
  91 + config = self._oc['config']
  92 + units = config.get_units()
  93 + kunits = list(units.keys())
  94 + kunit = kunits[0]
  95 + log.info(f"unit key = {kunit}")
  96 + unit = units[kunit]
  97 + self._unit = unit
  98 + # log.info(f"unit = {unit}")
  99 + agents = config.get_agents(kunit)
  100 + # log.info(f"agents = {agents}")
  101 + attrs = agents[agent_alias]
  102 + key = 'path_data_root'
  103 + if key in attrs:
  104 + self._path_data_root = attrs['path_data_root']
  105 + else:
  106 + # default with docker should be /home/pyros_user/app
  107 + self._path_data_root = os.environ['PROJECT_ROOT_PATH']
  108 + if self._path_data_root == None:
  109 + self._path_data_root = os.environ['PROJECT_ROOT_PATH']
  110 + log.info(f"path_data_root = {self._path_data_root}")
  111 +
  112 + # === Define and create the directories for the images to process
  113 + # place this paragraph in a common class for camera device and image processing
  114 + self._paths = {}
  115 + self._paths['ima_incoming'] = os.path.join(self._path_data_root,"data/images/incoming")
  116 + self._paths['ima_processed'] = os.path.join(self._path_data_root,"data/images/processed")
  117 + self._paths['ima_tmp'] = os.path.join(self._path_data_root,"data/images/tmp")
  118 + self._paths['ima_darks'] = os.path.join(self._path_data_root,"data/images/darks")
  119 + self._paths['ima_flats'] = os.path.join(self._path_data_root,"data/images/flats")
  120 + self._paths['ima_bias'] = os.path.join(self._path_data_root,"data/images/bias")
  121 + for path in self._paths.values():
  122 + os.makedirs(path, exist_ok = True)
  123 +
  124 + # === Define an image to test the processing and copy it in incoming directory
  125 + if self.TEST_MODE == True:
  126 + self._file_ima_test = os.path.join(self._path_data_root,"vendor/guitastro/test/data/m57.fit")
  127 + file_in = self._file_ima_test
  128 + file_out = f"{self._paths['ima_incoming']}/m57.fit"
  129 + shutil.copyfile(file_in, file_out)
  130 +
  131 + # === Instanciate an object Ima to make image processing
  132 + self._ima = guitastro.Ima()
  133 + home = guitastro.Home(self._unit['home'])
  134 +
  135 + # === Get longitude of the unit to generate the night yyyymmdd and subdirectories yyyy/mm/dd
  136 + longitude = home.longitude
  137 + log.info(f"Longitude={longitude}")
  138 + self._ima.longitude(longitude)
  139 + log.info("Init done with success")
  140 +
  141 + # === Status of routine processing
  142 + self._routine_running = self.RUNNING_NOTHING
  143 +
  144 + # = aklotz =====================================================
  145 +
87 146  
88 147 '''
89 148 # @override
... ... @@ -107,30 +166,140 @@ class AgentImagesProcessor(Agent):
107 166 def do_log(self):
108 167 super().do_log()
109 168  
110   - def apply_calibration_files(self):
111   - print("\n Starting calibration of new image files...\n")
112   - time.sleep(5)
  169 + def glob_images_to_process(self):
  170 +
  171 + # = aklotz =====================================================
  172 + # - glob the incoming directory:
  173 + fitsfiles = glob.glob(f"{self._paths['ima_incoming']}/*.fit")
  174 + # - Please sort list of files in increasing dates (TODO)
  175 + return fitsfiles
  176 +
  177 + def bias_correction(self):
  178 +
  179 + # - Search the bias
  180 + path_bias = os.path.join( self._paths['ima_bias'], self._date_night )
  181 + fitsbiasfiles = glob.glob(f"{path_bias}/*.fit")
  182 + log.info(f"fitsbiasfiles = {fitsbiasfiles}")
  183 + if len(fitsbiasfiles) > 0:
  184 +
  185 + # - Select the bias
  186 + pass
  187 +
  188 + def dark_correction(self):
  189 +
  190 + # - Search the dark
  191 + path_darks = os.path.join( self._paths['ima_darks'], self._date_night )
  192 + fitsdarkfiles = glob.glob(f"{path_darks}/*.fit")
  193 + log.info(f"fitsdarkfiles = {fitsdarkfiles}")
  194 + if len(fitsdarkfiles) > 0:
  195 +
  196 + # - Select two darks and compute the therm using exposure
  197 + # - Correction of dark
  198 + pass
113 199  
114   - # TODO:
  200 + def flat_correction(self):
115 201  
  202 + # - Search the flat
  203 + path_flats = os.path.join( self._paths['ima_flats'], self._date_night )
  204 + fitsflatfiles = glob.glob(f"{path_flats}/*.fit")
  205 + log.info(f"fitsflatfiles = {fitsflatfiles}")
  206 + if len(fitsflatfiles) > 0:
  207 + # - Select the flat (with the filter)
  208 + # - Correction of flat
  209 + pass
  210 +
  211 + def process_one_image(self, fitsfile: str):
  212 +
  213 + # = aklotz =====================================================
  214 +
  215 + # - Load file in memory
  216 + log.info("Load the file in memory")
  217 + f = self._ima.genename(self._ima.load(fitsfile))
  218 + # log.info(f"f={f}")
  219 +
  220 + # - Save as tmp
  221 + self._ima.path(self._paths['ima_tmp'])
  222 + log.info("Save the temporary file as tmp name")
  223 + self._ima.save("tmp")
  224 +
  225 + # - Load tmp and get infos
  226 + self._ima.load("tmp")
  227 + date_obs = self._ima.getkwd("DATE-OBS")
  228 + self._date_night = self._ima.get_night(date_obs)
  229 + log.info(f"Date_obs = {date_obs}")
  230 + log.info(f"Night = {self._date_night}")
  231 + exposure = self._ima.getkwd("EXPOSURE")
  232 + log.info(f"Exposure = {exposure}")
  233 +
  234 + # - Bias correction
  235 + self.bias_correction()
  236 +
  237 + # - Dark correction
  238 + self.dark_correction()
  239 +
  240 + # - Flat correction
  241 + self.flat_correction()
  242 +
  243 + # - Save tmp corrected by dark and flat
  244 + self._ima.path(self._paths['ima_tmp'])
  245 + self._ima.save("tmp")
  246 +
  247 + # - Inversion of mirrors or mirorxy (TODO)
  248 +
  249 + # - Cosmetic correction (TODO)
  250 +
  251 + # - WCS calibration (TODO)
  252 +
  253 + # - Prepare the output file name
  254 + log.info("Decode the filename")
  255 + fgen_in = f['genename'] + f['sep'] + f['indexes'][0] + f['suffix']
  256 + fext_in = f['file_extension']
  257 + fext_out = ".fits"
  258 +
  259 + # - Save in processed
  260 + yyyy = self._date_night[0:4]
  261 + mm = self._date_night[4:6]
  262 + dd = self._date_night[6:8]
  263 + path_processed = os.path.join( self._paths['ima_processed'], yyyy, mm, dd )
  264 + self._ima.path(path_processed)
  265 + fname_out = fgen_in + fext_out
  266 + fname = self._ima.save(fname_out)
  267 + log.info(f"Save the processed image {fname}")
  268 +
  269 + # - Delete the file in incoming directory
  270 + os.remove(fitsfile)
  271 + log.info(f"Delete the raw image {fitsfile}")
  272 +
  273 + # - Update the running state
  274 + self._routine_running = self.RUNNING_NOTHING
  275 +
  276 + # ---
  277 + # = aklotz =====================================================
  278 +
  279 + time.sleep(5)
116 280 print("\n ...End of images calibration\n")
117 281  
118 282 # Note : called by _routine_process() in Agent
119 283 # @override
120 284 def routine_process_before_body(self):
121   - print("The Observatory configuration :")
122   - self.show_config()
123 285 log.debug("in routine_process_before_body()")
124   - #self.apply_calibration_files()
125 286  
126 287 # Note : called by _routine_process() in Agent
127 288 # @override
128 289 def routine_process_after_body(self):
129   - print("The Observatory configuration :")
130   - #self.show_config()
131 290 log.debug("in routine_process_after_body()")
132   - self.apply_calibration_files()
133   -
  291 + if self._routine_running == self.RUNNING_NOTHING:
  292 + # Get files to process
  293 + fitsfiles = self.glob_images_to_process()
  294 + n = len(fitsfiles)
  295 + log.info(f"There are {n} images to process")
  296 + if n > 0:
  297 + # - We select the oldest image
  298 + fitsfile = fitsfiles[0]
  299 + log.info(f"Process the file {fitsfile}")
  300 + # - Thread TODO
  301 + self._routine_running = self.RUNNING_ONE_IMAGE_PROCESSING
  302 + self.process_one_image(fitsfile)
134 303  
135 304 '''
136 305 # @override
... ...