From 198132303d731ebe7a310c5a0780ce287b60c0de Mon Sep 17 00:00:00 2001 From: Alain Klotz Date: Fri, 22 Sep 2023 23:59:59 +0200 Subject: [PATCH] New DATA structure in two Agents. --- config/pyros_observatory/general/schemas/schema_observatory-2.0.yml | 14 ++++++++++++++ src/core/pyros_django/img_process/A_ImgProcessor.py | 15 ++++++++++----- src/core/pyros_django/scheduling/A_Scheduler.py | 109 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------- 3 files changed, 102 insertions(+), 36 deletions(-) diff --git a/config/pyros_observatory/general/schemas/schema_observatory-2.0.yml b/config/pyros_observatory/general/schemas/schema_observatory-2.0.yml index 55503f8..f6dcdac 100644 --- a/config/pyros_observatory/general/schemas/schema_observatory-2.0.yml +++ b/config/pyros_observatory/general/schemas/schema_observatory-2.0.yml @@ -135,6 +135,20 @@ schema;schema_FN_CONTEXTS: type: map required: False mapping: + pyros_cat: + type: map + required: False + mapping: + root_dir: + type: str + description: + type: str + extension: + type: str + naming: + type: str + pathnaming: + type: str pyros_skd: type: map required: False diff --git a/src/core/pyros_django/img_process/A_ImgProcessor.py b/src/core/pyros_django/img_process/A_ImgProcessor.py index f6c01b5..9c0d4bc 100755 --- a/src/core/pyros_django/img_process/A_ImgProcessor.py +++ b/src/core/pyros_django/img_process/A_ImgProcessor.py @@ -58,7 +58,7 @@ class A_ImgProcessor(Agent): "do_stop_current_processing" : (60, Agent.EXEC_MODE.SEQUENTIAL, ''), } - # Scenario to be executed + # Test scenario to be executed (option -t) # "self do_stop_current_processing" # AgentCmd.CMD_STATUS_CODE.CMD_EXECUTED _TEST_COMMANDS_LIST = [ @@ -119,13 +119,16 @@ class A_ImgProcessor(Agent): log.info(f"{longitude=}") self._ima.longitude(longitude) - # === Copy the channel file contexts into the Ima object + # === Copy and update the channel file contexts into the Ima object self._ima.fcontext_replace(self._fn) log.info(f"=== List of file name contexts available for the channel {channel['name']} ({channel['symbol']})") for fcname in self._ima.fcontexts: self._ima.fcontext = fcname + if self.is_in_test_mode(): + self._ima.rootdir = os.path.abspath(self._ima.rootdir.replace("/PRODUCTS/","/PRODUCTS/TESTS/")) + os.makedirs(self._ima.rootdir, exist_ok=True) log.info(f"{fcname} : {self._ima.fdescription} {self._ima.rootdir}/[{self._ima.pathing()}]/[{self._ima.naming()}]{self._ima.extension}") - + # === Status of routine processing self._routine_running = self.RUNNING_NOTHING log.info("Init done with success") @@ -514,7 +517,8 @@ class A_ImgProcessor(Agent): log.info(f"Test image night: {datedict['night']}") # --- Get other infos filter_symbol = "C" - id_sequence = 123456789 + kid = 110 + id_sequence = int("999" + f"{kid:07d}") # --- Rename the image file name when copying to be compatible with the img_L0 file context try: @@ -647,7 +651,8 @@ class A_ImgProcessor(Agent): att = at # --- Get other infos filter_symbol = "C" - id_sequence = 123452789 + kid = 210 + id_sequence = int("999" + f"{kid:07d}") # --- Save the image file name to be compatible with the img_L0 file context try: diff --git a/src/core/pyros_django/scheduling/A_Scheduler.py b/src/core/pyros_django/scheduling/A_Scheduler.py index 5f3cb48..005f605 100755 --- a/src/core/pyros_django/scheduling/A_Scheduler.py +++ b/src/core/pyros_django/scheduling/A_Scheduler.py @@ -53,7 +53,7 @@ import numpy as np class A_Scheduler(Agent): - DPRINT = False + DPRINT = True # - Sampling of the night arrays (bins/night) BINS_NIGHT = 86400 @@ -85,13 +85,13 @@ class A_Scheduler(Agent): "do_create_seq_1" : (60, Agent.EXEC_MODE.SEQUENTIAL, ''), } - # Scenario to be executed + # Test scenario to be executed (option -t) # "self do_stop_current_processing" # AgentCmd.CMD_STATUS_CODE.CMD_EXECUTED _TEST_COMMANDS_LIST = [ # Format : ("self cmd_name cmd_args", timeout, "expected_result", expected_status), (True, "self do_create_seq_1 6", 200, '', Agent.CMD_STATUS.CMD_EXECUTED), - #(True, "self do_stop asap", 500, "STOPPING", Agent.CMD_STATUS.CMD_EXECUTED), + (True, "self do_stop asap", 500, "STOPPING", Agent.CMD_STATUS.CMD_EXECUTED), ] """ @@ -126,11 +126,15 @@ class A_Scheduler(Agent): # === Get home (Home object) of current unit home = guitastro.Home(self._home) - # === Get all file contexts from pyros config + # === Get and update all file contexts from pyros config self._fn = self.config.fn - - # === Select the file contextof sequence files - self._fn.pathnaming("PyROS.seq.1") + log.info(f"=== List of file name contexts available for the unit") + for fcname in self._fn.fcontexts: + self._fn.fcontext = fcname + if self.is_in_test_mode(): + self._fn.rootdir = os.path.abspath(self._fn.rootdir.replace("/PRODUCTS/","/PRODUCTS/TESTS/")) + os.makedirs(self._fn.rootdir, exist_ok=True) + log.info(f"{fcname} : {self._fn.fdescription} {self._fn.rootdir}/[{self._fn.pathing()}]/[{self._fn.naming()}]{self._fn.extension}") # === Set longitude to sequence file context to manage the night subdirectories longitude = home.longitude @@ -167,7 +171,10 @@ class A_Scheduler(Agent): def do_create_seq_1(self, nb_seq:int): """Create sequences to debug """ - self._create_seq_1(nb_seq) + try: + self._create_seq_1(nb_seq) + except Exception as e: + self.dprint(f"ERROR {e}") def do_compute_schedule_1(self): """Compute a schedule @@ -182,7 +189,10 @@ class A_Scheduler(Agent): Columns are defined by the enum SEQ_* (see the python code itself). """ - self._compute_schedule_1() + try: + self._compute_schedule_1() + except Exception as e: + self.dprint(f"ERROR {e}") """ ================================================================= @@ -511,7 +521,7 @@ class A_Scheduler(Agent): v = PredictiveSchedule.objects.last() if v == None: v = PredictiveSchedule() - #print(f"{v=}") + #log.info(f"{v=}") v.scheduler_matrix = ouput_matrix v.save() # --- Save the numpy matrix in database (via Json) @@ -522,32 +532,68 @@ class A_Scheduler(Agent): v.save() # --- Update the running state self._routine_running = self.RUNNING_NOTHING - print(f"_compute_schedule_1 finished in {time.time() - t0:.2f} seconds") + log.info(f"_compute_schedule_1 finished in {time.time() - t0:.2f} seconds") - def _create_seq_1(self, nb_seq): + def _create_seq_1(self, nb_seq: int): t0 = time.time() self.dprint("Debut _create_seq_1") - seq_template = {'sequence': {'id': 4, 'start_expo_pref': 'IMMEDIATE', 'pyros_user': 2, 'scientific_program': 1, 'name': 'seq_20230628T102140', 'desc': None, 'last_modified_by': 2, 'is_alert': False, 'status': 'TBP', 'with_drift': False, 'priority': None, 'analysis_method': None, 'moon_min': None, 'alt_min': None, 'type': None, 'img_current': None, 'img_total': None, 'not_obs': False, 'obsolete': False, 'processing': False, 'flag': None, 'period': 1, 'start_date': datetime.datetime(2023, 6, 28, 10, 21, 40, tzinfo=zoneinfo.ZoneInfo(key='UTC')), 'end_date': datetime.datetime(2023, 6, 28, 10, 21, 40, 999640, tzinfo=datetime.timezone.utc), 'jd1': Decimal('0E-8'), 'jd2': Decimal('0E-8'), 'tolerance_before': '1s', 'tolerance_after': '1min', 'duration': -1.0, 'overhead': Decimal('0E-8'), 'submitted': False, 'config_attributes': {'tolerance_before': '1s', 'tolerance_after': '1min', 'target': 'RADEC 0H10M -15D', 'conformation': 'WIDE', 'layout': 'Altogether'}, 'ra': None, 'dec': None, 'complete': True, 'night_id': '20230627'}, 'albums': {'Altogether': {'plans': [{'id': 4, 'album': 4, 'duration': 0.0, 'nb_images': 1, 'config_attributes': {'binnings': {'binxy': [1, 1], 'readouttime': 6}, 'exposuretime': 1.0}, 'complete': True}]}}} + seq_template = {'sequence': {'id': 4, 'start_expo_pref': 'IMMEDIATE', 'pyros_user': 2, 'scientific_program': 1, 'name': 'seq_20230628T102140', 'desc': None, 'last_modified_by': 2, 'is_alert': False, 'status': 'TBP', 'with_drift': False, 'priority': None, 'analysis_method': None, 'moon_min': None, 'alt_min': None, 'type': None, 'img_current': None, 'img_total': None, 'not_obs': False, 'obsolete': False, 'processing': False, 'flag': None, 'period': 1, 'start_date': datetime.datetime(2023, 6, 28, 10, 21, 40, tzinfo=zoneinfo.ZoneInfo(key='UTC')), 'end_date': datetime.datetime(2023, 6, 28, 10, 21, 40, 999640, tzinfo=datetime.timezone.utc), 'jd1': Decimal('0E-8'), 'jd2': Decimal('0E-8'), 'tolerance_before': '1s', 'tolerance_after': '1min', 'duration': -1.0, 'overhead': Decimal('0E-8'), 'submitted': False, 'config_attributes': {'tolerance_before': '1s', 'tolerance_after': '1min', 'target': 'RADEC 0H10M -15D', 'conformation': 'WIDE', 'layout': 'Altogether'}, 'ra': None, 'dec': None, 'complete': True, 'night_id': '20230627'}, 'albums': {'Altogether': {'plans': [{'id': 4, 'album': 4, 'duration': 0.0, 'nb_fnges': 1, 'config_attributes': {'binnings': {'binxy': [1, 1], 'readouttime': 6}, 'exposuretime': 1.0}, 'complete': True}]}}} # decode general variables info a dict info info = self.get_infos() rootdir = info['rootdir'] subdir = info['subdir'] + # --- Prepare ephemeris object # TBD duskelev a parametrer from obsconfig (yml) duskelev = -7 eph = guitastro.Ephemeris() eph.set_home(self.config.getHome()) - print(f"Debut _create_seq_1 SUN {info['night']=}") - try: + + # --- Build the path and file name of the sun ephemeris + self._fn.fcontext = "pyros_eph" + fn_param = { + "period" : f"{info['period_id']}", + "date": info['night'], + "version": "1", + "unit": self.config.unit_name, + "target": "sun" + } + fname = self._fn.naming_set(fn_param) + eph_file = self._fn.joinabs(fname) + # --- Compute the sun ephemeris if needed. Save it as pickle. + if not os.path.exists(eph_file): ephem_sun = eph.target2night("sun", info['night'], None, None) - except Exception as e: - print(f"{e.message=} {e.args=}") - print(f"Debut _create_seq_1 MOON {eph.target2night=}") - ephem_moon = eph.target2night("moon", info['night'], None, None) + os.makedirs(os.path.dirname(eph_file), exist_ok=True) + pickle.dump(ephem_sun, open(eph_file,"wb")) + else: + ephem_sun = pickle.load(open(eph_file,"rb")) + self._fn.fcontext = "pyros_seq" + + # --- Build the path and file name of the moon ephemeris + self._fn.fcontext = "pyros_eph" + fn_param = { + "period" : f"{info['period_id']}", + "date": info['night'], + "version": "1", + "unit": self.config.unit_name, + "target": "moon" + } + fname = self._fn.naming_set(fn_param) + eph_file = self._fn.joinabs(fname) + # --- Compute the moon ephemeris if needed. Save it as pickle. + if not os.path.exists(eph_file): + ephem_moon = eph.target2night("moon", info['night'], None, None) + os.makedirs(os.path.dirname(eph_file), exist_ok=True) + pickle.dump(ephem_moon, open(eph_file,"wb")) + else: + ephem_moon = pickle.load(open(eph_file,"rb")) + self._fn.fcontext = "pyros_seq" + # --- Horizon (TBD get from config) - print("Debut _create_seq_1 Horizon") + self.dprint("Debut _create_seq_1 Horizon") hor = guitastro.Horizon(eph.home) hor.horizon_altaz = [ [0,0], [360,0] ] + # --- Delete all existing *.p and *.f files in the night directory fn_param = { "period" : f"{info['period_id']}", @@ -568,7 +614,8 @@ class A_Scheduler(Agent): for seq_dfile in seq_dfiles: #print(f":::.1 : os.remove {seq_dfile=}") os.remove(seq_dfile) - # --- + + # --- Create new sequences for k in range(nb_seq): #print("B"*20 + f" {info['operiod'].id} {info['night']} {k}") time.sleep(1) @@ -599,22 +646,21 @@ class A_Scheduler(Agent): seq['sequence']['duration'] = duration seq['sequence']['config_attributes']['target'] = target # --- Build the path and file name of the sequence file - fn_param["id_seq"] = k - print(f"{k} : {self._fn.fcontext=}") + fn_param["id_seq"] = int("999" + f"{k:07d}") + self.dprint(f"{k} : {self._fn.fcontext=}") self._fn.fname = self._fn.naming_set(fn_param) - print(f"{k} : {self._fn.fname=}") + self.dprint(f"{k} : {self._fn.fname=}") seq_file = self._fn.join(self._fn.fname) - print(f"{k} : {seq_file=}") + self.dprint(f"{k} : {seq_file=}") # --- Build the path and file name of the ephemeris file eph_file = f"{seq_file[:-2]}.f" # --- Create directory if it doesn't exist - print(f"{k} : {seq_file=}") + self.dprint(f"{k} : {seq_file=}") os.makedirs(os.path.dirname(seq_file), exist_ok=True) # --- Compute the ephemeris of the sequence and manage errors #print(f"{k} : TRY") errors = [] try: - # TODO remplacer les none par les fichiers pickle de ephem_sun & ephem_moon ephem = eph.target2night(seq["sequence"]["config_attributes"]["target"], info['night'], ephem_sun, ephem_moon, preference=seq['sequence']['start_expo_pref'], duskelev=duskelev, horizon=hor, duration=duration) except ValueError: errors.append("Target value is not valid") @@ -625,15 +671,16 @@ class A_Scheduler(Agent): if len(errors) == 0: pickle.dump(ephem, open(eph_file,"wb")) pickle.dump(seq, open(seq_file,"wb")) - #print(f"{errors=}") - #print("C"*20) - print(f"_create_seq_1 finished in {time.time() - t0:.2f} seconds") + #dprint(f"{errors=}") + #dprint("C"*20) + log.info(f"_create_seq_1 finished in {time.time() - t0:.2f} seconds") def load_sequence(self): sequence = "" return sequence def get_infos(self): + self._fn.fcontext = "pyros_seq" rootdir = self._fn.rootdir operiod = Period.objects.exploitation_period() if operiod == None: @@ -657,7 +704,7 @@ class A_Scheduler(Agent): def dprint(self, *args, **kwargs): if self.DPRINT: - print(*args, **kwargs) + log.info(*args, **kwargs) if __name__ == "__main__": -- libgit2 0.21.2