Commit d9da346b42ee86e4b7b148f9d2b60db6bd547344
Exists in
dev
Merge branch 'dev' of https://gitlab.irap.omp.eu/pyros-irap/pyros into dev
Showing
1 changed file
with
543 additions
and
96 deletions
Show diff stats
src/core/pyros_django/scheduling/A_Scheduler.py
| 1 | 1 | #!/usr/bin/env python3 |
| 2 | +# | |
| 3 | +# To launch this agent from the root of Pyros: | |
| 4 | +# | |
| 5 | +# Linux console: | |
| 6 | +# cd /srv/develop/pyros/docker | |
| 7 | +# ./PYROS_DOCKER_START.sh | |
| 8 | +# | |
| 9 | +# Launch from Power Shell: | |
| 10 | +# To go from docker to Powershell: pyros_user@ORION:~/app$ exit (or Ctrl+d) | |
| 11 | +# Prompt is now PS ...> | |
| 12 | +# cd \srv\develop\pyros | |
| 13 | +# .\PYROS -t new-start -o tnc -fg -a A_Scheduler | |
| 14 | +# | |
| 15 | +# Launch from docker: | |
| 16 | +# To go from Powershell to docker: PS ...> .\PYROS_DOCKER_SHELL | |
| 17 | +# Prompt is now pyros_user@ORION:~/app$ | |
| 18 | +# ./PYROS -t new-start -o tnc -fg -a A_Scheduler | |
| 19 | +# ./PYROS -d -t new-start -o tnc -fg -a A_Scheduler | |
| 20 | +# | |
| 21 | +# --------------------------------------------------- | |
| 2 | 22 | |
| 3 | 23 | import sys |
| 4 | -##import utils.Logger as L | |
| 5 | -#import threading #, multiprocessing, os | |
| 6 | 24 | import time |
| 25 | +import argparse | |
| 26 | +import os | |
| 27 | +import pickle | |
| 28 | +import socket | |
| 29 | +pwd = os.environ['PROJECT_ROOT_PATH'] | |
| 30 | +if pwd not in sys.path: | |
| 31 | + sys.path.append(pwd) | |
| 7 | 32 | |
| 8 | -#from django.db import transaction | |
| 9 | -#from common.models import Command | |
| 33 | +short_paths = ['src', 'src/core/pyros_django'] | |
| 34 | +for short_path in short_paths: | |
| 35 | + path = os.path.join(pwd, short_path) | |
| 36 | + if path not in sys.path: | |
| 37 | + sys.path.insert(0, path) | |
| 10 | 38 | |
| 11 | -sys.path.append("..") | |
| 12 | -sys.path.append("../../../..") | |
| 13 | -from src.core.pyros_django.majordome.agent.Agent import Agent, build_agent, log | |
| 14 | - | |
| 15 | -# PM 20190416 recycle code | |
| 16 | -#from common.models import * | |
| 39 | +from src.core.pyros_django.majordome.agent.Agent import Agent, build_agent, log, parse_args | |
| 17 | 40 | from seq_submit.models import Sequence |
| 41 | +from user_mgmt.models import Period, ScientificProgram, SP_Period | |
| 42 | +# = Specials | |
| 43 | +import glob | |
| 44 | +import shutil | |
| 45 | +import guitastro | |
| 46 | +import datetime | |
| 47 | +from decimal import Decimal | |
| 48 | +import zoneinfo | |
| 49 | +import numpy as np | |
| 18 | 50 | |
| 19 | -##log = L.setupLogger("AgentXTaskLogger", "AgentX") | |
| 20 | - | |
| 21 | - | |
| 22 | - | |
| 23 | -class A_Scheduler(Agent): | |
| 24 | - | |
| 51 | +class AgentScheduler(Agent): | |
| 25 | 52 | |
| 26 | - # FOR TEST ONLY | |
| 27 | - # Run this agent in simulator mode | |
| 28 | - TEST_MODE = False | |
| 29 | - # Run the assertion tests at the end | |
| 30 | - TEST_WITH_FINAL_TEST = True | |
| 31 | - TEST_MAX_DURATION_SEC = None | |
| 32 | - #TEST_MAX_DURATION_SEC = 120 | |
| 33 | - | |
| 34 | - # PM 20190416 fucking config path starting: /home/patrick/Dev/PYROS/start_agent.py agentM | |
| 35 | - ##_path_data = 'config' | |
| 36 | - _path_data = 'config/old_config' | |
| 53 | + DPRINT = False | |
| 54 | + | |
| 55 | + # - status of the sequence after schedule computation | |
| 56 | + SEQ_NOT_PROCESSED = 0 | |
| 57 | + SEQ_SCHEDULED = 1 | |
| 58 | + SEQ_SCHEDULED_OVER_QUOTA = 2 | |
| 59 | + SEQ_REJECTED_NO_QUOTA_ENOUGH = -1 | |
| 60 | + SEQ_REJECTED_NO_SLOT_AVAILABLE = -2 | |
| 61 | + | |
| 62 | + # - enum of the matrix line | |
| 63 | + SEQ_K = 0 | |
| 64 | + SEQ_SEQ_ID = 1 | |
| 65 | + SEQ_KOBS0 = 2 | |
| 66 | + SEQ_SP_ID = 3 | |
| 67 | + SEQ_PRIORITY = 4 | |
| 68 | + SEQ_DURATION = 5 | |
| 69 | + SEQ_STATUS = 6 | |
| 70 | + NB_SEQ = 7 | |
| 71 | + | |
| 72 | + # - All possible running states | |
| 73 | + RUNNING_NOTHING = 0 | |
| 74 | + RUNNING_SCHEDULE_PROCESSING = 1 | |
| 37 | 75 | |
| 38 | - log.debug("PLC instanciated") | |
| 76 | + _AGENT_SPECIFIC_COMMANDS = { | |
| 77 | + # Format : “cmd_name” : (timeout, exec_mode) | |
| 78 | + "do_compute_schedule_1" : (60, Agent.EXEC_MODE.SEQUENTIAL, ''), | |
| 79 | + "do_create_seq_1" : (60, Agent.EXEC_MODE.SEQUENTIAL, ''), | |
| 80 | + } | |
| 39 | 81 | |
| 40 | - _AGENT_SPECIFIC_COMMANDS = [ | |
| 41 | - #"do_replan", | |
| 42 | - "do_stop_replan", | |
| 82 | + # Scenario to be executed | |
| 83 | + # "self do_stop_current_processing" | |
| 84 | + # AgentCmd.CMD_STATUS_CODE.CMD_EXECUTED | |
| 85 | + _TEST_COMMANDS_LIST = [ | |
| 86 | + # Format : ("self cmd_name cmd_args", timeout, "expected_result", expected_status), | |
| 87 | + (True, "self do_create_seq_1 6", 200, '', Agent.CMD_STATUS.CMD_EXECUTED), | |
| 88 | + (True, "self do_stop asap", 500, "STOPPING", Agent.CMD_STATUS.CMD_EXECUTED), | |
| 43 | 89 | ] |
| 44 | 90 | |
| 45 | - | |
| 46 | 91 | """ |
| 47 | 92 | ================================================================= |
| 48 | - FUNCTIONS RUN INSIDE MAIN THREAD | |
| 93 | + Methods running inside main thread | |
| 49 | 94 | ================================================================= |
| 50 | 95 | """ |
| 51 | - # old config | |
| 52 | - # @override | |
| 53 | - #def __init__(self, name:str=None, config_filename=None, RUN_IN_THREAD=True): | |
| 54 | - # def __init__(self, config_filename=None, RUN_IN_THREAD=True): | |
| 55 | - # ##if name is None: name = self.__class__.__name__ | |
| 56 | - # super().__init__(config_filename, RUN_IN_THREAD) | |
| 57 | - | |
| 58 | - # new config (obsconfig) | |
| 59 | - #def __init__(self, name:str=None, RUN_IN_THREAD=True): | |
| 60 | - def __init__(self, name:str=None): | |
| 96 | + def __init__(self, name:str=None,simulated_computer=None): | |
| 61 | 97 | if name is None: |
| 62 | 98 | name = self.__class__.__name__ |
| 63 | - super().__init__() | |
| 64 | - #super().__init__(RUN_IN_THREAD) | |
| 99 | + super().__init__(simulated_computer=simulated_computer) | |
| 65 | 100 | |
| 66 | - # @override | |
| 67 | 101 | def _init(self): |
| 68 | 102 | super()._init() |
| 69 | - log.debug("end init()") | |
| 70 | - # --- Set the mode according the startmode value | |
| 71 | - ##agent_alias = self.__class__.__name__ | |
| 72 | - ##self.set_mode_from_config(agent_alias) | |
| 103 | + log.debug("end super init()") | |
| 104 | + log.info(f"self.TEST_MODE = {self.TEST_MODE}") | |
| 73 | 105 | |
| 74 | - ''' | |
| 75 | - # @override | |
| 76 | - def load_config(self): | |
| 77 | - super().load_config() | |
| 78 | - ''' | |
| 106 | + # === Get the config object | |
| 107 | + self.config = self._oc['config'] | |
| 108 | + self.pconfig = self._oc['pyros_config'] | |
| 109 | + # === Get agent_alias | |
| 110 | + hostname = socket.gethostname() | |
| 111 | + log.info(f"{hostname=}") | |
| 112 | + log.info(f"{self.name=}") | |
| 113 | + agent_alias = self.config.get_agent_real_name(self.name, hostname) | |
| 114 | + log.info(f"{agent_alias=}") | |
| 79 | 115 | |
| 80 | - ''' | |
| 81 | - # @override | |
| 82 | - def update_survey(self): | |
| 83 | - super().update_survey() | |
| 84 | - ''' | |
| 116 | + # === Get self._home of current unit | |
| 117 | + self._home = self.config.getHome() | |
| 118 | + home = guitastro.Home(self._home) | |
| 85 | 119 | |
| 86 | - ''' | |
| 87 | - # @override | |
| 88 | - def get_next_command(self): | |
| 89 | - return super().get_next_command() | |
| 90 | - ''' | |
| 120 | + self._fn = self.pconfig.fn | |
| 121 | + self._fn.pathnaming("PyROS.seq.1") | |
| 91 | 122 | |
| 92 | - # @override | |
| 93 | - def do_log(self): | |
| 94 | - super().do_log() | |
| 95 | - | |
| 96 | - def replan_sequences(self): | |
| 97 | - print("\n start of sequences (re-)planning...\n") | |
| 98 | - time.sleep(5) | |
| 99 | - sequences = Sequence.objects.filter(status="TBP") | |
| 100 | - print("List of sequences to be planned :") | |
| 101 | - for seq in sequences: | |
| 102 | - print('-', seq.name, '('+seq.status+') :') | |
| 103 | - print('-- albums : ', seq.albums.all()) | |
| 104 | - print('-- plans : ') | |
| 105 | - for album in seq.albums.all(): | |
| 106 | - print(album) | |
| 107 | - for plan in album.plans.all(): | |
| 108 | - print('plan id', plan.id) | |
| 109 | - | |
| 110 | - print("\n ...end of sequences (re-)planning\n") | |
| 123 | + # === Set longitude to ima object to generate the night yyyymmdd and subdirectories yyyy/mm/dd | |
| 124 | + longitude = home.longitude | |
| 125 | + log.info(f"{longitude=}") | |
| 126 | + self._fn.longitude(longitude) | |
| 127 | + | |
| 128 | + # === Status of routine processing | |
| 129 | + self._routine_running = self.RUNNING_NOTHING | |
| 130 | + log.debug("end init()") | |
| 111 | 131 | |
| 112 | 132 | # Note : called by _routine_process() in Agent |
| 113 | 133 | # @override |
| 114 | 134 | def _routine_process_iter_start_body(self): |
| 115 | - print("The Observatory configuration :") | |
| 116 | - self.show_config() | |
| 117 | 135 | log.debug("in routine_process_before_body()") |
| 118 | - #self.replan_sequences() | |
| 119 | 136 | |
| 120 | 137 | # Note : called by _routine_process() in Agent |
| 121 | 138 | # @override |
| 122 | 139 | def _routine_process_iter_end_body(self): |
| 123 | - print("The Observatory configuration :") | |
| 124 | - #self.show_config() | |
| 125 | 140 | log.debug("in routine_process_after_body()") |
| 126 | - self.replan_sequences() | |
| 141 | + # TODO EP est-ce utile ? | |
| 142 | + if self._routine_running == self.RUNNING_NOTHING: | |
| 143 | + # Get files to process | |
| 144 | + # - Thread TODO | |
| 145 | + self._routine_running = self.RUNNING_SCHEDULE_PROCESSING | |
| 146 | + self.do_compute_schedule_1() | |
| 127 | 147 | |
| 128 | - ''' | |
| 129 | - # @override | |
| 130 | - def exec_specific_cmd_end(self, cmd:Command, from_thread=True): | |
| 131 | - super().exec_specific_cmd_end(cmd, from_thread) | |
| 132 | - ''' | |
| 148 | + """ | |
| 149 | + ================================================================= | |
| 150 | + Methods of specific commands | |
| 151 | + ================================================================= | |
| 152 | + """ | |
| 153 | + | |
| 154 | + def do_ccreate_seq_1(self, nb_seq:int): | |
| 155 | + """Create sequences to debug | |
| 156 | + """ | |
| 157 | + self._create_seq_1(nb_seq) | |
| 158 | + | |
| 159 | + def do_compute_schedule_1(self): | |
| 160 | + """Compute a schedule | |
| 161 | + | |
| 162 | + According the current time, select the night directory. | |
| 163 | + List the *.p file list (.p for sequences) | |
| 164 | + Read the *.p, *.f file contents (.f for ephemeris) | |
| 165 | + Compute the schedule | |
| 166 | + | |
| 167 | + Output is a matrix to unpack in the database. | |
| 168 | + Each line of the matrix is a sequence | |
| 169 | + Columns are defined by the enum SEQ_* (see the python code itself). | |
| 170 | + | |
| 171 | + """ | |
| 172 | + self._compute_schedule_1() | |
| 173 | + | |
| 174 | + """ | |
| 175 | + ================================================================= | |
| 176 | + Methods called by commands or routine. Overload these methods | |
| 177 | + ================================================================= | |
| 178 | + # --- | |
| 179 | + # osp = ScientificProgram.objects.get(id=scientific_program_id) | |
| 180 | + # --- ospperiod is the SP object | |
| 181 | + # ospperiod = SP_Period.objects.get(period = period_id, scientific_program = osp) | |
| 182 | + # print(f"dir(ospperiod)={dir(ospperiod)}") | |
| 183 | + # dir(spperiod)=['DoesNotExist', | |
| 184 | + # 'IS_VALID', 'IS_VALID_ACCEPTED', 'IS_VALID_REJECTED', | |
| 185 | + # 'MultipleObjectsReturned', 'SP_Period_Guests', 'SP_Period_Users', | |
| 186 | + # 'STATUSES', 'STATUSES_ACCEPTED', 'STATUSES_DRAFT', | |
| 187 | + # 'STATUSES_EVALUATED', 'STATUSES_REJECTED', 'STATUSES_SUBMITTED', | |
| 188 | + # 'VISIBILITY_CHOICES', 'VISIBILITY_NO', 'VISIBILITY_YES', | |
| 189 | + # 'VOTES', 'VOTES_NO', 'VOTES_TO_DISCUSS', 'VOTES_YES', | |
| 190 | + # 'can_submit_sequence', 'check', 'clean', 'clean_fields', | |
| 191 | + # 'date_error_message', 'delete', 'from_db', 'full_clean', | |
| 192 | + # 'get_constraints', 'get_deferred_fields', 'get_is_valid_display', | |
| 193 | + # 'get_public_visibility_display', 'get_status_display', | |
| 194 | + # 'get_vote_referee1_display', 'get_vote_referee2_display', | |
| 195 | + # 'id', 'is_currently_active', 'is_valid', 'objects', | |
| 196 | + # 'over_quota_duration', 'over_quota_duration_allocated', | |
| 197 | + # 'over_quota_duration_remaining', 'period', 'period_id', | |
| 198 | + # 'pk', 'prepare_database_save', 'priority', 'public_visibility', | |
| 199 | + # 'quota_allocated', 'quota_minimal', 'quota_nominal', | |
| 200 | + # 'quota_remaining', 'reason_referee1', 'reason_referee2', | |
| 201 | + # 'referee1', 'referee1_id', 'referee2', 'referee2_id', | |
| 202 | + # 'refresh_from_db', 'save', 'save_base', 'scientific_program', | |
| 203 | + # 'scientific_program_id', 'serializable_value', 'status', 'token', | |
| 204 | + # 'token_allocated', 'token_remaining', 'unique_error_message', | |
| 205 | + # 'validate_constraints', 'validate_unique', 'vote_referee1', | |
| 206 | + # 'vote_referee2' | |
| 207 | + """ | |
| 208 | + | |
| 209 | + def _compute_schedule_1(self): | |
| 210 | + """Simple scheduler based on selection-insertion one state algorithm. | |
| 211 | + | |
| 212 | + Quotas are available only fo the night. | |
| 213 | + No token. | |
| 214 | + """ | |
| 215 | + t0 = time.time() | |
| 216 | + #self.DPRINT = True | |
| 217 | + # --- Get the incoming directory of the night | |
| 218 | + info = self.get_infos() | |
| 219 | + rootdir = info['rootdir'] | |
| 220 | + subdir = info['subdir'] | |
| 221 | + # --- Build the wildcard to list the sequences | |
| 222 | + wildcard = os.path.join(rootdir, subdir, "*.p") | |
| 223 | + self.dprint(f"{wildcard=}") | |
| 224 | + # --- List the sequences from the incoming directory | |
| 225 | + seqfiles = glob.glob(wildcard) | |
| 226 | + log.info(f"{len(seqfiles)} file sequences to process") | |
| 227 | + # --- Initialize the schedule | |
| 228 | + schedule = np.zeros(86400, dtype=int) -1 | |
| 229 | + schedule_binary = np.ones(86400, dtype=int) | |
| 230 | + # =================================================================== | |
| 231 | + # --- Loop over the sequences of the night to extract useful infos | |
| 232 | + # =================================================================== | |
| 233 | + self.dprint("\n" + "="*70 + f"\n=== Read {len(seqfiles)} sequence files of the night {info['night']}\n" + "="*70 + "\n") | |
| 234 | + sequence_infos = [] | |
| 235 | + # --- Initialize the list of scientific_program_ids | |
| 236 | + scientific_program_ids = [] | |
| 237 | + kseq = 0 | |
| 238 | + for seqfile in seqfiles: | |
| 239 | + # --- seqfile = sequence file name | |
| 240 | + kseq += 1 | |
| 241 | + sequence_info = {} | |
| 242 | + sequence_info['id'] = -1 # TBD replace by idseq of the database | |
| 243 | + sequence_info['seqfile'] = seqfile | |
| 244 | + sequence_info['error'] = "" | |
| 245 | + sequence_info['kobs0'] = -1 | |
| 246 | + # --- ephfile = ephemeris file name | |
| 247 | + ephfile = os.path.splitext(seqfile)[0] + ".f" | |
| 248 | + # --- If ephemeris file exists, read files | |
| 249 | + if os.path.exists(ephfile): | |
| 250 | + self.dprint(f"Read file {seqfile}") | |
| 251 | + # --- seq_info = sequence dictionary | |
| 252 | + # --- eph_info = ephemeris dictionary | |
| 253 | + seq_info = pickle.load(open(seqfile,"rb")) | |
| 254 | + #print("="*20 + "\n" + f"{seq_info=}") | |
| 255 | + eph_info = pickle.load(open(ephfile,"rb")) | |
| 256 | + #print("="*20 + "\n" + f"{eph_info=}") | |
| 257 | + # --- | |
| 258 | + param = self._fn.naming_get(seqfile) | |
| 259 | + sequence_info['id'] = int(param['id_seq']) | |
| 260 | + # --- scientific_program_id is an integer | |
| 261 | + scientific_program_id = seq_info['sequence']['scientific_program'] | |
| 262 | + # --- Dictionary of informations about the sequence | |
| 263 | + sequence_info['seq_dico'] = seq_info # useful for duration | |
| 264 | + # --- Search the last time when the start of the sequence is observable (visibility > 0) | |
| 265 | + visibility_duration = eph_info['visibility_duration'] | |
| 266 | + kobss = np.where(visibility_duration > 0) | |
| 267 | + kobss = list(kobss[0]) | |
| 268 | + if len(kobss) == 0: | |
| 269 | + self.dprint(" Sequence has no visibility") | |
| 270 | + sequence_info['error'] = f"Sequence has no visibility_duration" | |
| 271 | + sequence_infos.append(sequence_info) | |
| 272 | + continue | |
| 273 | + kobs0 = kobss[0] | |
| 274 | + sequence_info['kobs0'] = kobs0 | |
| 275 | + sequence_info['visibility'] = eph_info['visibility'] # total slots | |
| 276 | + sequence_info['visibility_duration'] = visibility_duration # total slots - duration | |
| 277 | + sequence_info['duration'] = seq_info['sequence']['duration'] | |
| 278 | + sequence_info['scientific_program_id'] = scientific_program_id | |
| 279 | + self.dprint(f" {scientific_program_id=} range to start={len(kobss)}") | |
| 280 | + if scientific_program_id not in scientific_program_ids: | |
| 281 | + scientific_program_ids.append(scientific_program_id) | |
| 282 | + else: | |
| 283 | + sequence_info['error'] = f"File {ephfile} not exists" | |
| 284 | + sequence_infos.append(sequence_info) | |
| 285 | + | |
| 286 | + # =================================================================== | |
| 287 | + # --- Get informations of priority and quota from scientific programs | |
| 288 | + # =================================================================== | |
| 289 | + self.dprint("\n" + "="*70 + f"\n=== Get information from {len(scientific_program_ids)} scientific programs of the night\n" + "="*70 + "\n") | |
| 290 | + scientific_program_infos = {} | |
| 291 | + period_id = info['operiod'].id | |
| 292 | + self.dprint(f"{scientific_program_ids=}") | |
| 293 | + for scientific_program_id in scientific_program_ids: | |
| 294 | + scientific_program_info = {} | |
| 295 | + try: | |
| 296 | + osp = ScientificProgram.objects.get(id=scientific_program_id) | |
| 297 | + # --- ospperiod is the SP object | |
| 298 | + ospperiod = SP_Period.objects.get(period = period_id, scientific_program = osp) | |
| 299 | + scientific_program_info['priority'] = ospperiod.priority | |
| 300 | + scientific_program_info['over_quota_duration'] = ospperiod.over_quota_duration | |
| 301 | + scientific_program_info['over_quota_duration_allocated'] = ospperiod.over_quota_duration_allocated | |
| 302 | + scientific_program_info['over_quota_duration_remaining'] = ospperiod.over_quota_duration_remaining | |
| 303 | + scientific_program_info['quota_allocated'] = ospperiod.quota_allocated | |
| 304 | + scientific_program_info['quota_minimal'] = ospperiod.quota_minimal | |
| 305 | + scientific_program_info['quota_nominal'] = ospperiod.quota_nominal | |
| 306 | + scientific_program_info['quota_remaining'] = ospperiod.quota_remaining | |
| 307 | + scientific_program_info['token_allocated'] = ospperiod.token_allocated | |
| 308 | + scientific_program_info['token_remaining'] = ospperiod.token_allocated | |
| 309 | + except: | |
| 310 | + # --- simulation | |
| 311 | + scientific_program_info['priority'] = 0 | |
| 312 | + if scientific_program_info['priority'] == 0: | |
| 313 | + # --- simulation | |
| 314 | + priority = 50 + scientific_program_id*5 | |
| 315 | + scientific_program_info['priority'] = priority | |
| 316 | + scientific_program_info['quota_allocated'] = 12000 | |
| 317 | + scientific_program_info['quota_remaining'] = 12000 | |
| 318 | + scientific_program_infos[str(scientific_program_id)] = scientific_program_info | |
| 319 | + self.dprint(f"{scientific_program_id=} priority={scientific_program_info['priority']} quota={scientific_program_info['quota_remaining']}") | |
| 320 | + | |
| 321 | + # =================================================================== | |
| 322 | + # --- Build the numpy matrix seqs to make rapid computations | |
| 323 | + # =================================================================== | |
| 324 | + self.dprint("\n" + "="*70 + f"\n=== Build the matrix for scheduling {len(sequence_infos)} sequences\n" + "="*70 + "\n") | |
| 325 | + self.dprint("Order ID_seq K_start ID_sp Priority Duration Status\n") | |
| 326 | + nseq = len(sequence_infos) | |
| 327 | + if nseq == 0: | |
| 328 | + self._routine_running = self.RUNNING_NOTHING | |
| 329 | + return | |
| 330 | + seqs = np.zeros((nseq, self.NB_SEQ), dtype=int) | |
| 331 | + k = 0 | |
| 332 | + for sequence_info in sequence_infos: | |
| 333 | + if 'scientific_program_id' not in sequence_info.keys(): | |
| 334 | + self.dprint(f"No scientific program for ID sequence {sequence_info['id']}") | |
| 335 | + continue | |
| 336 | + scientific_program_id = sequence_info['scientific_program_id'] | |
| 337 | + scientific_program_info = scientific_program_infos[str(scientific_program_id)] | |
| 338 | + priority = scientific_program_info['priority'] | |
| 339 | + # Order of the following list refers to the enum | |
| 340 | + seq = [ k, sequence_info['id'], sequence_info['kobs0'], scientific_program_id, priority, int(np.ceil(sequence_info['duration'])), self.SEQ_NOT_PROCESSED ] | |
| 341 | + self.dprint(f"{seq=}") | |
| 342 | + seqs[k] = seq | |
| 343 | + k += 1 | |
| 344 | + seqs = seqs[:k] | |
| 345 | + # --- Save the matrix sequence | |
| 346 | + #print(f"{seqs=}") | |
| 347 | + fpathname = os.path.join(rootdir, subdir, "scheduler_seq_matrix1.txt") | |
| 348 | + np.savetxt(fpathname, seqs) | |
| 349 | + | |
| 350 | + # =================================================================== | |
| 351 | + # --- Compute the matrix seq_sorteds (priority and chronology) | |
| 352 | + # =================================================================== | |
| 353 | + self.dprint("\n" + "="*70 + "\n=== Sort the matrix for scheduling by priority and chronology\n" + "="*70 + "\n") | |
| 354 | + # --- Sort the matrix sequence: priority=SEQ_PRIORITY (decreasing -1) and then chronology=SEQ_KOBS0 (increasing +1) | |
| 355 | + seq_sorteds = seqs[np.lexsort(([1,-1]*seqs[:,[self.SEQ_KOBS0, self.SEQ_PRIORITY]]).T)] | |
| 356 | + # --- Save the matrix sequence | |
| 357 | + self.dprint("Order ID_seq K_start ID_sp Priority Duration Status\n") | |
| 358 | + self.dprint(f"{seq_sorteds=}") | |
| 359 | + fpathname = os.path.join(rootdir, subdir, "scheduler_seq_matrix2.txt") | |
| 360 | + np.savetxt(fpathname, seq_sorteds) | |
| 361 | + | |
| 362 | + # =================================================================== | |
| 363 | + # --- Insert sequences in the schedule. Respecting priority and quota | |
| 364 | + # =================================================================== | |
| 365 | + self.dprint("\n" + "="*70 + "\n=== Insertion of the sequences in the schedule respecting priority and quota\n" + "="*70 + "\n") | |
| 366 | + for seq in seq_sorteds: | |
| 367 | + # --- Unpack the sequence | |
| 368 | + k, sequence_id, kobs0, scientific_program_id, priority, duration, seq_status = seq | |
| 369 | + | |
| 370 | + # --- Get the quota remaining of the scientific program | |
| 371 | + quota_remaining = scientific_program_infos[str(scientific_program_id)]['quota_remaining'] | |
| 372 | + self.dprint('-'*70 + "\n" + f"Process {sequence_id=} {kobs0=} {duration=} sp_id={scientific_program_id} {quota_remaining=}") | |
| 373 | + | |
| 374 | + # --- Verify if duration < quota_remaining | |
| 375 | + if duration > quota_remaining: | |
| 376 | + # --- No remaining quota to insert this sequence | |
| 377 | + self.dprint(f"{sequence_id=} cannot be inserted because no quota enough") | |
| 378 | + seqs[k][self.SEQ_STATUS] = self.SEQ_REJECTED_NO_QUOTA_ENOUGH | |
| 379 | + continue | |
| 380 | + | |
| 381 | + # --- Compute the remaining visibility and list (k1s) of the best observation start | |
| 382 | + # =0 if not possible to start observation | |
| 383 | + # =value with the highest value for the best observation start | |
| 384 | + | |
| 385 | + # --- Visibility*schedule_binary are transformed into binary | |
| 386 | + sequence_info = sequence_infos[k] | |
| 387 | + vis_binarys = sequence_info['visibility'].copy() * schedule_binary | |
| 388 | + vis_binarys[vis_binarys > 0] = 1 | |
| 389 | + | |
| 390 | + # --- Cumulative sum + offset by -duration to prepare the start_binary computation | |
| 391 | + obs_starts = np.cumsum(vis_binarys) | |
| 392 | + obs_ends = obs_starts.copy() | |
| 393 | + obs_ends[0:-duration] = obs_ends[duration:] | |
| 394 | + obs_ends[-duration:] = 0 | |
| 395 | + | |
| 396 | + # --- Difference and binarisation to get starts with duration | |
| 397 | + start_binary = obs_ends - obs_starts | |
| 398 | + start_binary[start_binary < duration] = 0 | |
| 399 | + start_binary[start_binary == duration] = 1 | |
| 400 | + | |
| 401 | + # --- Compute the remaining visibility (float) | |
| 402 | + remaining_visibility = sequence_info['visibility'] * start_binary | |
| 403 | + | |
| 404 | + # --- Check the remaining visibility | |
| 405 | + if np.sum(remaining_visibility) == 0: | |
| 406 | + # --- No remaining slot to insert this sequence | |
| 407 | + self.dprint(f"{sequence_id=} cannot inserted because no more slots available") | |
| 408 | + seqs[k][self.SEQ_STATUS] = self.SEQ_REJECTED_NO_SLOT_AVAILABLE | |
| 409 | + continue | |
| 410 | + | |
| 411 | + # --- From the index of the highest value of remaining visibility to the index of the lowest value of remaining visibility | |
| 412 | + k1s = np.flip(np.argsort(remaining_visibility)) | |
| 413 | + self.dprint(f"{k1s=} => Start elevation {sequence_info['visibility'][k1s[0]]:+.2f}") | |
| 414 | + | |
| 415 | + # --- Get k1 as the highest value of remaining visibility | |
| 416 | + k1 = k1s[0] | |
| 417 | + k2 = k1 + duration | |
| 418 | + self.dprint(f"{k} : {sequence_id=} {scientific_program_id=} {priority=} inserted in the slot {k1=} {k2=} (remaining {quota_remaining - duration} s)") | |
| 419 | + | |
| 420 | + # --- Update the seqs matrix | |
| 421 | + seqs[k][self.SEQ_STATUS] = self.SEQ_SCHEDULED | |
| 422 | + | |
| 423 | + # --- Update the schedule arrays | |
| 424 | + schedule[k1:k2] = sequence_id | |
| 425 | + schedule_binary[k1:k2] = 0 | |
| 426 | + | |
| 427 | + # --- Update the scientific program dict | |
| 428 | + quota_remaining -= duration | |
| 429 | + scientific_program_infos[str(scientific_program_id)]['quota_remaining'] = quota_remaining | |
| 430 | + | |
| 431 | + # =================================================================== | |
| 432 | + # --- Insert sequences in the schedule. Respecting priority but over quota | |
| 433 | + # =================================================================== | |
| 434 | + # self.dprint("\n" + "="*70 + "\n=== Insertion of the sequences in the schedule respecting priority but over quota\n" + "="*70 + "\n") | |
| 435 | + # TBD | |
| 436 | + # where are remaining free slots | |
| 437 | + # scan sequences to insert in these free slots | |
| 438 | + | |
| 439 | + # =================================================================== | |
| 440 | + # --- Save the schedule | |
| 441 | + # =================================================================== | |
| 442 | + self.dprint("\n" + "="*70 + "\n=== Save the schedule\n" + "="*70 + "\n") | |
| 443 | + self.dprint("Order ID_seq K_start ID_sp Priority Duration Status\n") | |
| 444 | + self.dprint(f"{seqs=}") | |
| 445 | + fpathname = os.path.join(rootdir, subdir, "scheduler_schedule.txt") | |
| 446 | + np.savetxt(fpathname, np.array([schedule, schedule_binary]).T) | |
| 447 | + # --- Update the running state | |
| 448 | + self._routine_running = self.RUNNING_NOTHING | |
| 449 | + print(f"_compute_schedule_1 finished in {time.time() - t0:.2f} seconds") | |
| 133 | 450 | |
| 451 | + def _create_seq_1(self, nb_seq): | |
| 452 | + t0 = time.time() | |
| 453 | + self.dprint("Debut _create_seq_1") | |
| 454 | + seq_template = {'sequence': {'id': 4, 'start_expo_pref': 'IMMEDIATE', 'pyros_user': 2, 'scientific_program': 1, 'name': 'seq_20230628T102140', 'desc': None, 'last_modified_by': 2, 'is_alert': False, 'status': 'TBP', 'with_drift': False, 'priority': None, 'analysis_method': None, 'moon_min': None, 'alt_min': None, 'type': None, 'img_current': None, 'img_total': None, 'not_obs': False, 'obsolete': False, 'processing': False, 'flag': None, 'period': 1, 'start_date': datetime.datetime(2023, 6, 28, 10, 21, 40, tzinfo=zoneinfo.ZoneInfo(key='UTC')), 'end_date': datetime.datetime(2023, 6, 28, 10, 21, 40, 999640, tzinfo=datetime.timezone.utc), 'jd1': Decimal('0E-8'), 'jd2': Decimal('0E-8'), 'tolerance_before': '1s', 'tolerance_after': '1min', 'duration': -1.0, 'overhead': Decimal('0E-8'), 'submitted': False, 'config_attributes': {'tolerance_before': '1s', 'tolerance_after': '1min', 'target': 'RADEC 0H10M -15D', 'conformation': 'WIDE', 'layout': 'Altogether'}, 'ra': None, 'dec': None, 'complete': True, 'night_id': '20230627'}, 'albums': {'Altogether': {'plans': [{'id': 4, 'album': 4, 'duration': 0.0, 'nb_images': 1, 'config_attributes': {'binnings': {'binxy': [1, 1], 'readouttime': 6}, 'exposuretime': 1.0}, 'complete': True}]}}} | |
| 455 | + info = self.get_infos() | |
| 456 | + rootdir = info['rootdir'] | |
| 457 | + subdir = info['subdir'] | |
| 458 | + # --- Prepare ephemeris object | |
| 459 | + # TBD duskelev a parametrer from obsconfig (yml) | |
| 460 | + duskelev = -7 | |
| 461 | + eph = guitastro.Ephemeris() | |
| 462 | + eph.set_home(self.config.getHome()) | |
| 463 | + #print("Debut _create_seq_1 SUN") | |
| 464 | + ephem_sun = eph.target2night("sun", info['night'], None, None) | |
| 465 | + #print("Debut _create_seq_1 MOON") | |
| 466 | + ephem_moon = eph.target2night("moon", info['night'], None, None) | |
| 467 | + # --- Horizon (TBD get from config) | |
| 468 | + #print("Debut _create_seq_1 Horizon") | |
| 469 | + hor = guitastro.Horizon(eph.home) | |
| 470 | + hor.horizon_altaz = [ [0,0], [360,0] ] | |
| 471 | + # --- Delete all existing *.p and *.f files in the night directory | |
| 472 | + fn_param = { | |
| 473 | + "period" : f"{info['period_id']}", | |
| 474 | + "version": "1", | |
| 475 | + "unit": self.config.unit_name, | |
| 476 | + "date": info['night'], | |
| 477 | + "id_seq": 0 | |
| 478 | + } | |
| 479 | + fname = self._fn.naming_set(fn_param) | |
| 480 | + self.dprint(f":: {fname=}") | |
| 481 | + seq_file = self._fn.join(fname) | |
| 482 | + path_night = os.path.dirname(seq_file) | |
| 483 | + cards = ['*.p', '*.f'] | |
| 484 | + for card in cards: | |
| 485 | + wildcard = os.path.join(path_night, card) | |
| 486 | + seq_dfiles = glob.glob(wildcard) | |
| 487 | + #print(f"::: {seq_dfiles=}") | |
| 488 | + for seq_dfile in seq_dfiles: | |
| 489 | + #print(f":::.1 : os.remove {seq_dfile=}") | |
| 490 | + os.remove(seq_dfile) | |
| 491 | + # --- | |
| 492 | + for k in range(nb_seq): | |
| 493 | + #print("B"*20 + f" {info['operiod'].id} {info['night']} {k}") | |
| 494 | + time.sleep(1) | |
| 495 | + seq = seq_template.copy() | |
| 496 | + seq['sequence']['period'] = info['operiod'].id # int | |
| 497 | + seq['sequence']['night_id'] = info['night'] # str | |
| 498 | + seq['sequence']['config_attributes']['target'] = k # int | |
| 499 | + # --- | |
| 500 | + start_expo_pref = "BESTELEV" #"IMMEDIATE" | |
| 501 | + scientific_program = int(k/2) | |
| 502 | + start_date = datetime.datetime(2023, 6, 28, 10, 21, 40) | |
| 503 | + end_date = datetime.datetime(2023, 6, 28, 10, 21, 40, 999640, tzinfo=datetime.timezone.utc) | |
| 504 | + jd1 = Decimal('0E-8') | |
| 505 | + jd2 = Decimal('0E-8') | |
| 506 | + tolerance_before = '1s' | |
| 507 | + tolerance_after = '1min' | |
| 508 | + duration = 3000.0 | |
| 509 | + target = f"RADEC {k}h {10+2*k}d" | |
| 510 | + # --- | |
| 511 | + seq['sequence']['start_expo_pref'] = start_expo_pref | |
| 512 | + seq['sequence']['scientific_program'] = scientific_program | |
| 513 | + seq['sequence']['start_date'] = start_date | |
| 514 | + seq['sequence']['end_date'] = end_date | |
| 515 | + seq['sequence']['jd1'] = jd1 | |
| 516 | + seq['sequence']['jd2'] = jd2 | |
| 517 | + seq['sequence']['tolerance_before'] = tolerance_before | |
| 518 | + seq['sequence']['tolerance_after'] = tolerance_after | |
| 519 | + seq['sequence']['duration'] = duration | |
| 520 | + seq['sequence']['config_attributes']['target'] = target | |
| 521 | + # --- Build the path and file name of the sequence file | |
| 522 | + fn_param["id_seq"] = k | |
| 523 | + #print(f"{k} : {self._fn.fcontext=}") | |
| 524 | + self._fn.fname = self._fn.naming_set(fn_param) | |
| 525 | + #print(f"{k} : {self._fn.fname=}") | |
| 526 | + seq_file = self._fn.join(self._fn.fname) | |
| 527 | + #print(f"{k} : {seq_file=}") | |
| 528 | + # --- Build the path and file name of the ephemeris file | |
| 529 | + eph_file = f"{seq_file[:-2]}.f" | |
| 530 | + # --- Create directory if it doesn't exist | |
| 531 | + #print(f"{k} : {seq_file=}") | |
| 532 | + os.makedirs(os.path.dirname(seq_file), exist_ok=True) | |
| 533 | + # --- Compute the ephemeris of the sequence and manage errors | |
| 534 | + #print(f"{k} : TRY") | |
| 535 | + errors = [] | |
| 536 | + try: | |
| 537 | + # TODO remplacer les none par les fichiers pickle de ephem_sun & ephem_moon | |
| 538 | + ephem = eph.target2night(seq["sequence"]["config_attributes"]["target"], info['night'], ephem_sun, ephem_moon, preference=seq['sequence']['start_expo_pref'], duskelev=duskelev, horizon=hor, duration=duration) | |
| 539 | + except ValueError: | |
| 540 | + errors.append("Target value is not valid") | |
| 541 | + except guitastro.ephemeris.EphemerisException as ephemException: | |
| 542 | + errors.append(str(ephemException)) | |
| 543 | + if len(errors) == 0 and np.sum(ephem["visibility"]) == 0 : | |
| 544 | + errors.append("Target is not visible.") | |
| 545 | + if len(errors) == 0: | |
| 546 | + pickle.dump(ephem, open(eph_file,"wb")) | |
| 547 | + pickle.dump(seq, open(seq_file,"wb")) | |
| 548 | + #print(f"{errors=}") | |
| 549 | + #print("C"*20) | |
| 550 | + print(f"_create_seq_1 finished in {time.time() - t0:.2f} seconds") | |
| 551 | + | |
| 552 | + def load_sequence(self): | |
| 553 | + sequence = "" | |
| 554 | + return sequence | |
| 134 | 555 | |
| 556 | + def get_infos(self): | |
| 557 | + rootdir = self._fn.rootdir | |
| 558 | + operiod = Period.objects.exploitation_period() | |
| 559 | + if operiod == None: | |
| 560 | + log.info("No period valid in the database") | |
| 561 | + self._routine_running = self.RUNNING_NOTHING | |
| 562 | + return | |
| 563 | + period_id = str(operiod.id) | |
| 564 | + if len(str(operiod.id)) < 3: | |
| 565 | + while len(period_id) < 3: | |
| 566 | + period_id = "0" + period_id | |
| 567 | + period_id = "P" + period_id | |
| 568 | + night_id = self._fn.date2night("now") | |
| 569 | + subdir = os.path.join(period_id, night_id) | |
| 570 | + dico = {} | |
| 571 | + dico['rootdir'] = rootdir | |
| 572 | + dico['subdir'] = subdir | |
| 573 | + dico['operiod'] = operiod # object | |
| 574 | + dico['period_id'] = period_id # str formated (P000) | |
| 575 | + dico['night'] = night_id # str (YYYYMMDD) | |
| 576 | + return dico | |
| 577 | + | |
| 578 | + def dprint(self, *args, **kwargs): | |
| 579 | + if self.DPRINT: | |
| 580 | + print(*args, **kwargs) | |
| 581 | + | |
| 135 | 582 | if __name__ == "__main__": |
| 136 | 583 | |
| 137 | - agent = build_agent(A_Scheduler) | |
| 584 | + agent = build_agent(AgentScheduler) | |
| 138 | 585 | print(agent) |
| 139 | 586 | agent.run() | ... | ... |