Merge branch 'dev' of https://gitlab.irap.omp.eu/pyros-irap/pyros into dev

Alexis Koralewski
2 parents d51648f0 def6f098
Showing 3 changed files with 88 additions and 32 deletions Show diff stats
src/core/pyros_django/pyros/settings.py
src/core/pyros_django/scheduling/A_Scheduler.py
src/core/pyros_django/scheduling/models.py
@@ -168,7 +168,7 @@ try:
 except:
     set_environment_variables_if_not_configured(ENV_PATH,ENV_SAMPLE_PATH)
-ALLOWED_HOSTS = ['localhost', '127.0.0.1', 'pyros.irap.omp.eu', 'astroguita.hd.free.fr', '0.0.0.0']
+ALLOWED_HOSTS = ['localhost', '127.0.0.1', 'pyros.irap.omp.eu', 'astroguita.freeboxos.fr', '0.0.0.0']
 # defining variables when using Docker
 if WITH_DOCKER:
     ALLOWED_HOSTS.append('0.0.0.0')
@@ -32,13 +32,14 @@ if pwd not in sys.path:
 short_paths = ['src', 'src/core/pyros_django']
 for short_path in short_paths:
-    path = os.path.join(pwd, short_path)
+    path = os.path.abspath(os.path.join(pwd, short_path))
     if path not in sys.path:
         sys.path.insert(0, path)
 from src.core.pyros_django.majordome.agent.Agent import Agent, build_agent, log, parse_args
 from seq_submit.models import Sequence
 from user_mgmt.models import Period, ScientificProgram, SP_Period
+from scheduling.models import PredictiveSchedule, EffectiveSchedule
 # = Specials
 import glob
 import shutil
@@ -85,7 +86,7 @@ class A_Scheduler(Agent):
     _TEST_COMMANDS_LIST = [
         # Format : ("self cmd_name cmd_args", timeout, "expected_result", expected_status),
         (True, "self do_create_seq_1 6", 200, '', Agent.CMD_STATUS.CMD_EXECUTED),
-        (True, "self do_stop asap", 500, "STOPPING", Agent.CMD_STATUS.CMD_EXECUTED),
+        #(True, "self do_stop asap", 500, "STOPPING", Agent.CMD_STATUS.CMD_EXECUTED),
     ]
     """
@@ -128,6 +129,7 @@ class A_Scheduler(Agent):
         # === Status of routine processing
         self._routine_running = self.RUNNING_NOTHING
         log.debug("end init()")
+        ##### TBD suppress redondant paths in print(f"=>=>=> {sys.path=}")
     # Note : called by _routine_process() in Agent
     # @override
@@ -151,7 +153,7 @@ class A_Scheduler(Agent):
     =================================================================
     """
-    def do_ccreate_seq_1(self, nb_seq:int):
+    def do_create_seq_1(self, nb_seq:int):
         """Create sequences to debug
         """
         self._create_seq_1(nb_seq)
@@ -213,7 +215,7 @@ class A_Scheduler(Agent):
         No token.
         """
         t0 = time.time()
-        #self.DPRINT = True
+        self.DPRINT = True
         # --- Get the incoming directory of the night
         info = self.get_infos()
         rootdir = info['rootdir']
@@ -225,8 +227,30 @@ class A_Scheduler(Agent):
         seqfiles = glob.glob(wildcard)
         log.info(f"{len(seqfiles)} file sequences to process")
         # --- Initialize the schedule
-        schedule = np.zeros(86400, dtype=int) -1
+        schedule_sequence_id = np.zeros(86400, dtype=int) -1
         schedule_binary = np.ones(86400, dtype=int)
+        schedule_visibility = np.zeros(86400, dtype=float)
+        schedule_order = np.zeros(86400, dtype=int) -1
+        schedule_jd = np.zeros(86400, dtype=float)
+        schedule_scientific_programm_id = np.zeros(86400, dtype=int) -1
+        # --- Get the numpy matrix in database (via Json)
+        try:
+            # --- Get the effective schedule matrix
+            input_matrix = EffectiveSchedule.objects.last().conv_numpy()
+            # --- Get the effective schedule arrays
+            schedule_eff_jd, schedule_eff_binary, schedule_eff_sequence_id, schedule_eff_scientific_programm_id, schedule_eff_order, schedule_eff_visibility = input_matrix
+            # --- Get the index in the night
+            night, index = self._fn.date2night("now", 86400)
+            schedule_sequence_id[0:index] = schedule_eff_sequence_id[0:index]
+            schedule_binary[0:index] = schedule_eff_binary[0:index]
+            schedule_visibility[0:index] = schedule_eff_visibility[0:index]
+            schedule_order[0:index] = schedule_eff_order[0:index]
+            schedule_jd[0:index] = schedule_eff_jd[0:index]
+            schedule_scientific_programm_id[0:index] = schedule_eff_scientific_programm_id[0:index]  
+        except:
+            pass
+        #print(f"{schedule_jd=}")
+
         # ===================================================================
         # --- Loop over the sequences of the night to extract useful infos
         # ===================================================================
@@ -282,6 +306,10 @@ class A_Scheduler(Agent):
             else:
                 sequence_info['error'] = f"File {ephfile} not exists"
             sequence_infos.append(sequence_info)
+        try:
+            schedule_jd = eph_info['jd']
+        except:
+            pass
         # ===================================================================
         # --- Get informations of priority and quota from scientific programs
@@ -337,7 +365,7 @@ class A_Scheduler(Agent):
             scientific_program_info = scientific_program_infos[str(scientific_program_id)]
             priority = scientific_program_info['priority']
             # Order of the following list refers to the enum
-            seq = [ k, sequence_info['id'], sequence_info['kobs0'], scientific_program_id, priority, int(np.ceil(sequence_info['duration'])), self.SEQ_NOT_PROCESSED ] 
+            seq = [ k, sequence_info['id'], sequence_info['kobs0'], scientific_program_id, priority, int(np.ceil(sequence_info['duration'])), self.SEQ_NOT_PROCESSED] 
             self.dprint(f"{seq=}")
             seqs[k] = seq
             k += 1
@@ -363,9 +391,12 @@ class A_Scheduler(Agent):
         # --- Insert sequences in the schedule. Respecting priority and quota
         # ===================================================================
         self.dprint("\n" + "="*70 + "\n=== Insertion of the sequences in the schedule respecting priority and quota\n" + "="*70 + "\n")
-        for seq in seq_sorteds:
+        kseq_sorted = -1
+        for seq_sorted in seq_sorteds:
+            kseq_sorted += 1
+
             # --- Unpack the sequence
-            k, sequence_id, kobs0, scientific_program_id, priority, duration, seq_status = seq
+            k, sequence_id, kobs0, scientific_program_id, priority, duration, seq_status = seq_sorted
             # --- Get the quota remaining of the scientific program
             quota_remaining = scientific_program_infos[str(scientific_program_id)]['quota_remaining']
@@ -421,12 +452,16 @@ class A_Scheduler(Agent):
             seqs[k][self.SEQ_STATUS] = self.SEQ_SCHEDULED
             # --- Update the schedule arrays
-            schedule[k1:k2] = sequence_id
+            schedule_sequence_id[k1:k2] = sequence_id
             schedule_binary[k1:k2] = 0
+            schedule_visibility[k1:k2] = sequence_info['visibility'][k1:k2]
+            schedule_order[k1:k2] = kseq_sorted
+            schedule_scientific_programm_id[k1:k2] = scientific_program_id
             # --- Update the scientific program dict
             quota_remaining -= duration
             scientific_program_infos[str(scientific_program_id)]['quota_remaining'] = quota_remaining
+                        
         # ===================================================================
         # --- Insert sequences in the schedule. Respecting priority but over quota
@@ -442,8 +477,25 @@ class A_Scheduler(Agent):
         self.dprint("\n" + "="*70 + "\n=== Save the schedule\n" + "="*70 + "\n")
         self.dprint("Order ID_seq K_start ID_sp Priority Duration Status\n")
         self.dprint(f"{seqs=}")
+        # --- Prepare the output matrix
+        ouput_matrix = np.array([schedule_jd, schedule_binary, schedule_sequence_id, schedule_scientific_programm_id, schedule_order, schedule_visibility])
+        # --- Save the numpy matrix in ASCII
         fpathname = os.path.join(rootdir, subdir, "scheduler_schedule.txt")
-        np.savetxt(fpathname, np.array([schedule, schedule_binary]).T)
+        np.savetxt(fpathname, ouput_matrix.T)
+        # --- Save the numpy matrix in database (via Json)
+        try:
+            v = PredictiveSchedule.objects.last()
+        except:
+            v = PredictiveSchedule()
+        v.scheduler_matrix = ouput_matrix
+        v.save()
+        # --- Save the numpy matrix in database (via Json)
+        try:
+            v = EffectiveSchedule.objects.last()
+        except:
+            v = EffectiveSchedule()
+        v.scheduler_matrix = ouput_matrix
+        v.save()
         # --- Update the running state
         self._routine_running = self.RUNNING_NOTHING
         print(f"_compute_schedule_1 finished in {time.time() - t0:.2f} seconds")
@@ -452,6 +504,7 @@ class A_Scheduler(Agent):
         t0 = time.time()
         self.dprint("Debut _create_seq_1")
         seq_template = {'sequence': {'id': 4, 'start_expo_pref': 'IMMEDIATE', 'pyros_user': 2, 'scientific_program': 1, 'name': 'seq_20230628T102140', 'desc': None, 'last_modified_by': 2, 'is_alert': False, 'status': 'TBP', 'with_drift': False, 'priority': None, 'analysis_method': None, 'moon_min': None, 'alt_min': None, 'type': None, 'img_current': None, 'img_total': None, 'not_obs': False, 'obsolete': False, 'processing': False, 'flag': None, 'period': 1, 'start_date': datetime.datetime(2023, 6, 28, 10, 21, 40, tzinfo=zoneinfo.ZoneInfo(key='UTC')), 'end_date': datetime.datetime(2023, 6, 28, 10, 21, 40, 999640, tzinfo=datetime.timezone.utc), 'jd1': Decimal('0E-8'), 'jd2': Decimal('0E-8'), 'tolerance_before': '1s', 'tolerance_after': '1min', 'duration': -1.0, 'overhead': Decimal('0E-8'), 'submitted': False, 'config_attributes': {'tolerance_before': '1s', 'tolerance_after': '1min', 'target': 'RADEC 0H10M -15D', 'conformation': 'WIDE', 'layout': 'Altogether'}, 'ra': None, 'dec': None, 'complete': True, 'night_id': '20230627'}, 'albums': {'Altogether': {'plans': [{'id': 4, 'album': 4, 'duration': 0.0, 'nb_images': 1, 'config_attributes': {'binnings': {'binxy': [1, 1], 'readouttime': 6}, 'exposuretime': 1.0}, 'complete': True}]}}}
+        # decode general variables info a dict info 
         info = self.get_infos()
         rootdir = info['rootdir']
         subdir = info['subdir']
@@ -460,12 +513,15 @@ class A_Scheduler(Agent):
         duskelev = -7
         eph = guitastro.Ephemeris()
         eph.set_home(self.config.getHome())
-        #print("Debut _create_seq_1 SUN")
-        ephem_sun = eph.target2night("sun", info['night'], None, None)
-        #print("Debut _create_seq_1 MOON")
+        print(f"Debut _create_seq_1 SUN {info['night']=}")
+        try:
+            ephem_sun = eph.target2night("sun", info['night'], None, None)
+        except Exception as e:
+            print(f"{e.message=} {e.args=}")
+        print(f"Debut _create_seq_1 MOON {eph.target2night=}")
         ephem_moon = eph.target2night("moon", info['night'], None, None)
         # --- Horizon (TBD get from config)
-        #print("Debut _create_seq_1 Horizon")
+        print("Debut _create_seq_1 Horizon")
         hor = guitastro.Horizon(eph.home)
         hor.horizon_altaz = [ [0,0], [360,0] ]
         # --- Delete all existing *.p and *.f files in the night directory
@@ -520,15 +576,15 @@ class A_Scheduler(Agent):
             seq['sequence']['config_attributes']['target'] = target
             # --- Build the path and file name of the sequence file
             fn_param["id_seq"] = k
-            #print(f"{k} : {self._fn.fcontext=}")
+            print(f"{k} : {self._fn.fcontext=}")
             self._fn.fname = self._fn.naming_set(fn_param)
-            #print(f"{k} : {self._fn.fname=}")
+            print(f"{k} : {self._fn.fname=}")
             seq_file = self._fn.join(self._fn.fname)
-            #print(f"{k} : {seq_file=}")
+            print(f"{k} : {seq_file=}")
             # --- Build the path and file name of the ephemeris file
             eph_file = f"{seq_file[:-2]}.f"
             # --- Create directory if it doesn't exist
-            #print(f"{k} : {seq_file=}")
+            print(f"{k} : {seq_file=}")
             os.makedirs(os.path.dirname(seq_file), exist_ok=True)
             # --- Compute the ephemeris of the sequence and manage errors
             #print(f"{k} : TRY")
@@ -7,6 +7,7 @@ import numpy as np
 import json
 from json import JSONEncoder
 import numpy
+import ast
 class NumpyArrayEncoder(JSONEncoder):
     def default(self, obj):
@@ -15,40 +16,39 @@ class NumpyArrayEncoder(JSONEncoder):
         return JSONEncoder.default(self, obj)
+    
 class EffectiveSchedule(models.Model):
     scheduler_matrix = JSONField(blank=True, null=True)    
-
+    
+    
     def save(self, *args, **kwargs):
         # Transform numpy matrix to JSON
         scheduler_matrix_as_json = json.dumps(self.scheduler_matrix, cls=NumpyArrayEncoder)
         self.scheduler_matrix = scheduler_matrix_as_json
         super(EffectiveSchedule, self).save(*args, **kwargs)
-    def get(self, *args, **kwargs):
+    def conv_numpy(self):
         # Transform JSON to numpy matrix 
         matrix_as_json = json.loads(self.scheduler_matrix)
-
-        self.scheduler_matrix = numpy.asarray(matrix_as_json["array"]) 
-        return super().get(**kwargs)
-    
+      
+        return numpy.array(matrix_as_json)
+        
 class PredictiveSchedule(models.Model):
     scheduler_matrix = JSONField(blank=True, null=True)    
-
+    
     def save(self, *args, **kwargs):
         # Transform numpy matrix to JSON
         scheduler_matrix_as_json = json.dumps(self.scheduler_matrix, cls=NumpyArrayEncoder)
         self.scheduler_matrix = scheduler_matrix_as_json
         super(PredictiveSchedule, self).save(*args, **kwargs)
-    def get(self, *args, **kwargs):
+    def conv_numpy(self, *args, **kwargs):
         # Transform JSON to numpy matrix 
         matrix_as_json = json.loads(self.scheduler_matrix)
-
-        self.scheduler_matrix = numpy.asarray(matrix_as_json["array"]) 
-        return super().get(**kwargs)
-    
-
+     
+        return numpy.array(matrix_as_json) 
+         
 class SchedulerHistory(EffectiveSchedule):
     night_datetime = models.DateTimeField(blank=True, null=True)
 \ No newline at end of file