New DATA structure in two Agents.

Alain Klotz
1 parent be9ae145
Showing 3 changed files with 102 additions and 36 deletions Show diff stats
config/pyros_observatory/general/schemas/schema_observatory-2.0.yml
src/core/pyros_django/img_process/A_ImgProcessor.py
src/core/pyros_django/scheduling/A_Scheduler.py
@@ -135,6 +135,20 @@ schema;schema_FN_CONTEXTS:
   type: map
   required: False
   mapping:
+    pyros_cat:
+      type: map 
+      required: False
+      mapping:
+        root_dir:
+          type: str
+        description:
+          type: str
+        extension:
+          type: str 
+        naming: 
+          type: str
+        pathnaming:
+          type: str 
     pyros_skd:
       type: map 
       required: False
@@ -58,7 +58,7 @@ class A_ImgProcessor(Agent):
         "do_stop_current_processing" : (60, Agent.EXEC_MODE.SEQUENTIAL, ''),
     }
  
-    # Scenario to be executed
+    # Test scenario to be executed (option -t)
     # "self do_stop_current_processing"
     # AgentCmd.CMD_STATUS_CODE.CMD_EXECUTED
     _TEST_COMMANDS_LIST = [
@@ -119,13 +119,16 @@ class A_ImgProcessor(Agent):
         log.info(f"{longitude=}")
         self._ima.longitude(longitude)
  
-        # === Copy the channel file contexts into the Ima object
+        # === Copy and update the channel file contexts into the Ima object
         self._ima.fcontext_replace(self._fn)
         log.info(f"=== List of file name contexts available for the channel {channel['name']} ({channel['symbol']})")
         for fcname in self._ima.fcontexts:
             self._ima.fcontext = fcname
+            if self.is_in_test_mode():
+                self._ima.rootdir = os.path.abspath(self._ima.rootdir.replace("/PRODUCTS/","/PRODUCTS/TESTS/"))
+            os.makedirs(self._ima.rootdir, exist_ok=True)
             log.info(f"{fcname} : {self._ima.fdescription} {self._ima.rootdir}/[{self._ima.pathing()}]/[{self._ima.naming()}]{self._ima.extension}")
-
+        
         # === Status of routine processing
         self._routine_running = self.RUNNING_NOTHING
         log.info("Init done with success")
@@ -514,7 +517,8 @@ class A_ImgProcessor(Agent):
         log.info(f"Test image night: {datedict['night']}")
         # --- Get other infos
         filter_symbol = "C"
-        id_sequence = 123456789
+        kid = 110
+        id_sequence = int("999" + f"{kid:07d}")
  
         # --- Rename the image file name when copying to be compatible with the img_L0 file context
         try:
@@ -647,7 +651,8 @@ class A_ImgProcessor(Agent):
             att = at
         # --- Get other infos
         filter_symbol = "C"
-        id_sequence = 123452789
+        kid = 210
+        id_sequence = int("999" + f"{kid:07d}")
  
         # --- Save the image file name to be compatible with the img_L0 file context
         try:
@@ -53,7 +53,7 @@ import numpy as np
  
 class A_Scheduler(Agent):
  
-    DPRINT = False
+    DPRINT = True
  
     # - Sampling of the night arrays (bins/night)
     BINS_NIGHT = 86400
@@ -85,13 +85,13 @@ class A_Scheduler(Agent):
         "do_create_seq_1" : (60, Agent.EXEC_MODE.SEQUENTIAL, ''),
     }
  
-    # Scenario to be executed
+    # Test scenario to be executed (option -t)
     # "self do_stop_current_processing"
     # AgentCmd.CMD_STATUS_CODE.CMD_EXECUTED
     _TEST_COMMANDS_LIST = [
         # Format : ("self cmd_name cmd_args", timeout, "expected_result", expected_status),
         (True, "self do_create_seq_1 6", 200, '', Agent.CMD_STATUS.CMD_EXECUTED),
-        #(True, "self do_stop asap", 500, "STOPPING", Agent.CMD_STATUS.CMD_EXECUTED),
+        (True, "self do_stop asap", 500, "STOPPING", Agent.CMD_STATUS.CMD_EXECUTED),
     ]
  
     """
@@ -126,11 +126,15 @@ class A_Scheduler(Agent):
         # === Get home (Home object) of current unit
         home = guitastro.Home(self._home)
  
-        # === Get all file contexts from pyros config
+        # === Get and update all file contexts from pyros config
         self._fn = self.config.fn
-
-        # === Select the file contextof sequence files
-        self._fn.pathnaming("PyROS.seq.1")
+        log.info(f"=== List of file name contexts available for the unit")
+        for fcname in self._fn.fcontexts:
+            self._fn.fcontext = fcname
+            if self.is_in_test_mode():
+                self._fn.rootdir = os.path.abspath(self._fn.rootdir.replace("/PRODUCTS/","/PRODUCTS/TESTS/"))
+            os.makedirs(self._fn.rootdir, exist_ok=True)
+            log.info(f"{fcname} : {self._fn.fdescription} {self._fn.rootdir}/[{self._fn.pathing()}]/[{self._fn.naming()}]{self._fn.extension}")
  
         # === Set longitude to sequence file context to manage the night subdirectories
         longitude = home.longitude
@@ -167,7 +171,10 @@ class A_Scheduler(Agent):
     def do_create_seq_1(self, nb_seq:int):
         """Create sequences to debug
         """
-        self._create_seq_1(nb_seq)
+        try:
+            self._create_seq_1(nb_seq)
+        except Exception as e:
+            self.dprint(f"ERROR {e}")
  
     def do_compute_schedule_1(self):
         """Compute a schedule
@@ -182,7 +189,10 @@ class A_Scheduler(Agent):
         Columns are defined by the enum SEQ_* (see the python code itself).
  
         """
-        self._compute_schedule_1()
+        try:
+            self._compute_schedule_1()
+        except Exception as e:
+            self.dprint(f"ERROR {e}")
  
     """
     =================================================================
@@ -511,7 +521,7 @@ class A_Scheduler(Agent):
         v = PredictiveSchedule.objects.last()
         if v == None:
             v = PredictiveSchedule()
-        #print(f"{v=}")
+        #log.info(f"{v=}")
         v.scheduler_matrix = ouput_matrix
         v.save()
         # --- Save the numpy matrix in database (via Json)
@@ -522,32 +532,68 @@ class A_Scheduler(Agent):
         v.save()
         # --- Update the running state
         self._routine_running = self.RUNNING_NOTHING
-        print(f"_compute_schedule_1 finished in {time.time() - t0:.2f} seconds")
+        log.info(f"_compute_schedule_1 finished in {time.time() - t0:.2f} seconds")
  
-    def _create_seq_1(self, nb_seq):
+    def _create_seq_1(self, nb_seq: int):
         t0 = time.time()
         self.dprint("Debut _create_seq_1")
-        seq_template = {'sequence': {'id': 4, 'start_expo_pref': 'IMMEDIATE', 'pyros_user': 2, 'scientific_program': 1, 'name': 'seq_20230628T102140', 'desc': None, 'last_modified_by': 2, 'is_alert': False, 'status': 'TBP', 'with_drift': False, 'priority': None, 'analysis_method': None, 'moon_min': None, 'alt_min': None, 'type': None, 'img_current': None, 'img_total': None, 'not_obs': False, 'obsolete': False, 'processing': False, 'flag': None, 'period': 1, 'start_date': datetime.datetime(2023, 6, 28, 10, 21, 40, tzinfo=zoneinfo.ZoneInfo(key='UTC')), 'end_date': datetime.datetime(2023, 6, 28, 10, 21, 40, 999640, tzinfo=datetime.timezone.utc), 'jd1': Decimal('0E-8'), 'jd2': Decimal('0E-8'), 'tolerance_before': '1s', 'tolerance_after': '1min', 'duration': -1.0, 'overhead': Decimal('0E-8'), 'submitted': False, 'config_attributes': {'tolerance_before': '1s', 'tolerance_after': '1min', 'target': 'RADEC 0H10M -15D', 'conformation': 'WIDE', 'layout': 'Altogether'}, 'ra': None, 'dec': None, 'complete': True, 'night_id': '20230627'}, 'albums': {'Altogether': {'plans': [{'id': 4, 'album': 4, 'duration': 0.0, 'nb_images': 1, 'config_attributes': {'binnings': {'binxy': [1, 1], 'readouttime': 6}, 'exposuretime': 1.0}, 'complete': True}]}}}
+        seq_template = {'sequence': {'id': 4, 'start_expo_pref': 'IMMEDIATE', 'pyros_user': 2, 'scientific_program': 1, 'name': 'seq_20230628T102140', 'desc': None, 'last_modified_by': 2, 'is_alert': False, 'status': 'TBP', 'with_drift': False, 'priority': None, 'analysis_method': None, 'moon_min': None, 'alt_min': None, 'type': None, 'img_current': None, 'img_total': None, 'not_obs': False, 'obsolete': False, 'processing': False, 'flag': None, 'period': 1, 'start_date': datetime.datetime(2023, 6, 28, 10, 21, 40, tzinfo=zoneinfo.ZoneInfo(key='UTC')), 'end_date': datetime.datetime(2023, 6, 28, 10, 21, 40, 999640, tzinfo=datetime.timezone.utc), 'jd1': Decimal('0E-8'), 'jd2': Decimal('0E-8'), 'tolerance_before': '1s', 'tolerance_after': '1min', 'duration': -1.0, 'overhead': Decimal('0E-8'), 'submitted': False, 'config_attributes': {'tolerance_before': '1s', 'tolerance_after': '1min', 'target': 'RADEC 0H10M -15D', 'conformation': 'WIDE', 'layout': 'Altogether'}, 'ra': None, 'dec': None, 'complete': True, 'night_id': '20230627'}, 'albums': {'Altogether': {'plans': [{'id': 4, 'album': 4, 'duration': 0.0, 'nb_fnges': 1, 'config_attributes': {'binnings': {'binxy': [1, 1], 'readouttime': 6}, 'exposuretime': 1.0}, 'complete': True}]}}}
         # decode general variables info a dict info 
         info = self.get_infos()
         rootdir = info['rootdir']
         subdir = info['subdir']
+        
         # --- Prepare ephemeris object
         # TBD duskelev a parametrer from obsconfig (yml)
         duskelev = -7
         eph = guitastro.Ephemeris()
         eph.set_home(self.config.getHome())
-        print(f"Debut _create_seq_1 SUN {info['night']=}")
-        try:
+        
+        # --- Build the path and file name of the sun ephemeris
+        self._fn.fcontext = "pyros_eph"
+        fn_param = {
+            "period" : f"{info['period_id']}",
+            "date": info['night'],
+            "version": "1",
+            "unit": self.config.unit_name,
+            "target": "sun"
+        }
+        fname = self._fn.naming_set(fn_param)
+        eph_file = self._fn.joinabs(fname)
+        # --- Compute the sun ephemeris if needed. Save it as pickle.
+        if not os.path.exists(eph_file):
             ephem_sun = eph.target2night("sun", info['night'], None, None)
-        except Exception as e:
-            print(f"{e.message=} {e.args=}")
-        print(f"Debut _create_seq_1 MOON {eph.target2night=}")
-        ephem_moon = eph.target2night("moon", info['night'], None, None)
+            os.makedirs(os.path.dirname(eph_file), exist_ok=True)
+            pickle.dump(ephem_sun, open(eph_file,"wb"))
+        else:
+            ephem_sun = pickle.load(open(eph_file,"rb"))
+        self._fn.fcontext = "pyros_seq"
+
+        # --- Build the path and file name of the moon ephemeris
+        self._fn.fcontext = "pyros_eph"
+        fn_param = {
+            "period" : f"{info['period_id']}",
+            "date": info['night'],
+            "version": "1",
+            "unit": self.config.unit_name,
+            "target": "moon"
+        }
+        fname = self._fn.naming_set(fn_param)
+        eph_file = self._fn.joinabs(fname)
+        # --- Compute the moon ephemeris if needed. Save it as pickle.
+        if not os.path.exists(eph_file):
+            ephem_moon = eph.target2night("moon", info['night'], None, None)
+            os.makedirs(os.path.dirname(eph_file), exist_ok=True)
+            pickle.dump(ephem_moon, open(eph_file,"wb"))
+        else:
+            ephem_moon = pickle.load(open(eph_file,"rb"))
+        self._fn.fcontext = "pyros_seq"
+
         # --- Horizon (TBD get from config)
-        print("Debut _create_seq_1 Horizon")
+        self.dprint("Debut _create_seq_1 Horizon")
         hor = guitastro.Horizon(eph.home)
         hor.horizon_altaz = [ [0,0], [360,0] ]
+        
         # --- Delete all existing *.p and *.f files in the night directory
         fn_param = {
             "period" : f"{info['period_id']}",
@@ -568,7 +614,8 @@ class A_Scheduler(Agent):
             for seq_dfile in seq_dfiles:
                 #print(f":::.1 : os.remove {seq_dfile=}")
                 os.remove(seq_dfile)
-        # ---
+                
+        # --- Create new sequences
         for k in range(nb_seq):
             #print("B"*20 + f" {info['operiod'].id} {info['night']} {k}")
             time.sleep(1)
@@ -599,22 +646,21 @@ class A_Scheduler(Agent):
             seq['sequence']['duration'] = duration
             seq['sequence']['config_attributes']['target'] = target
             # --- Build the path and file name of the sequence file
-            fn_param["id_seq"] = k
-            print(f"{k} : {self._fn.fcontext=}")
+            fn_param["id_seq"] = int("999" + f"{k:07d}")
+            self.dprint(f"{k} : {self._fn.fcontext=}")
             self._fn.fname = self._fn.naming_set(fn_param)
-            print(f"{k} : {self._fn.fname=}")
+            self.dprint(f"{k} : {self._fn.fname=}")
             seq_file = self._fn.join(self._fn.fname)
-            print(f"{k} : {seq_file=}")
+            self.dprint(f"{k} : {seq_file=}")
             # --- Build the path and file name of the ephemeris file
             eph_file = f"{seq_file[:-2]}.f"
             # --- Create directory if it doesn't exist
-            print(f"{k} : {seq_file=}")
+            self.dprint(f"{k} : {seq_file=}")
             os.makedirs(os.path.dirname(seq_file), exist_ok=True)
             # --- Compute the ephemeris of the sequence and manage errors
             #print(f"{k} : TRY")
             errors = []
             try:
-                # TODO remplacer les none par les fichiers pickle de ephem_sun & ephem_moon
                 ephem = eph.target2night(seq["sequence"]["config_attributes"]["target"], info['night'], ephem_sun, ephem_moon, preference=seq['sequence']['start_expo_pref'], duskelev=duskelev, horizon=hor, duration=duration)
             except ValueError:
                 errors.append("Target value is not valid")
@@ -625,15 +671,16 @@ class A_Scheduler(Agent):
             if len(errors) == 0:
                 pickle.dump(ephem, open(eph_file,"wb"))
                 pickle.dump(seq, open(seq_file,"wb"))
-            #print(f"{errors=}")
-            #print("C"*20)
-        print(f"_create_seq_1 finished in {time.time() - t0:.2f} seconds")
+            #dprint(f"{errors=}")
+            #dprint("C"*20)
+        log.info(f"_create_seq_1 finished in {time.time() - t0:.2f} seconds")
  
     def load_sequence(self):
         sequence = ""
         return sequence
  
     def get_infos(self):
+        self._fn.fcontext = "pyros_seq"
         rootdir = self._fn.rootdir
         operiod = Period.objects.exploitation_period()
         if operiod == None:
@@ -657,7 +704,7 @@ class A_Scheduler(Agent):
  
     def dprint(self, *args, **kwargs):
         if self.DPRINT:
-            print(*args, **kwargs)
+            log.info(*args, **kwargs)
  
 if __name__ == "__main__":