Commit d9da346b42ee86e4b7b148f9d2b60db6bd547344
Exists in
dev
Merge branch 'dev' of https://gitlab.irap.omp.eu/pyros-irap/pyros into dev
Showing
1 changed file
with
543 additions
and
96 deletions
Show diff stats
src/core/pyros_django/scheduling/A_Scheduler.py
1 | #!/usr/bin/env python3 | 1 | #!/usr/bin/env python3 |
2 | +# | ||
3 | +# To launch this agent from the root of Pyros: | ||
4 | +# | ||
5 | +# Linux console: | ||
6 | +# cd /srv/develop/pyros/docker | ||
7 | +# ./PYROS_DOCKER_START.sh | ||
8 | +# | ||
9 | +# Launch from Power Shell: | ||
10 | +# To go from docker to Powershell: pyros_user@ORION:~/app$ exit (or Ctrl+d) | ||
11 | +# Prompt is now PS ...> | ||
12 | +# cd \srv\develop\pyros | ||
13 | +# .\PYROS -t new-start -o tnc -fg -a A_Scheduler | ||
14 | +# | ||
15 | +# Launch from docker: | ||
16 | +# To go from Powershell to docker: PS ...> .\PYROS_DOCKER_SHELL | ||
17 | +# Prompt is now pyros_user@ORION:~/app$ | ||
18 | +# ./PYROS -t new-start -o tnc -fg -a A_Scheduler | ||
19 | +# ./PYROS -d -t new-start -o tnc -fg -a A_Scheduler | ||
20 | +# | ||
21 | +# --------------------------------------------------- | ||
2 | 22 | ||
3 | import sys | 23 | import sys |
4 | -##import utils.Logger as L | ||
5 | -#import threading #, multiprocessing, os | ||
6 | import time | 24 | import time |
25 | +import argparse | ||
26 | +import os | ||
27 | +import pickle | ||
28 | +import socket | ||
29 | +pwd = os.environ['PROJECT_ROOT_PATH'] | ||
30 | +if pwd not in sys.path: | ||
31 | + sys.path.append(pwd) | ||
7 | 32 | ||
8 | -#from django.db import transaction | ||
9 | -#from common.models import Command | 33 | +short_paths = ['src', 'src/core/pyros_django'] |
34 | +for short_path in short_paths: | ||
35 | + path = os.path.join(pwd, short_path) | ||
36 | + if path not in sys.path: | ||
37 | + sys.path.insert(0, path) | ||
10 | 38 | ||
11 | -sys.path.append("..") | ||
12 | -sys.path.append("../../../..") | ||
13 | -from src.core.pyros_django.majordome.agent.Agent import Agent, build_agent, log | ||
14 | - | ||
15 | -# PM 20190416 recycle code | ||
16 | -#from common.models import * | 39 | +from src.core.pyros_django.majordome.agent.Agent import Agent, build_agent, log, parse_args |
17 | from seq_submit.models import Sequence | 40 | from seq_submit.models import Sequence |
41 | +from user_mgmt.models import Period, ScientificProgram, SP_Period | ||
42 | +# = Specials | ||
43 | +import glob | ||
44 | +import shutil | ||
45 | +import guitastro | ||
46 | +import datetime | ||
47 | +from decimal import Decimal | ||
48 | +import zoneinfo | ||
49 | +import numpy as np | ||
18 | 50 | ||
19 | -##log = L.setupLogger("AgentXTaskLogger", "AgentX") | ||
20 | - | ||
21 | - | ||
22 | - | ||
23 | -class A_Scheduler(Agent): | ||
24 | - | 51 | +class AgentScheduler(Agent): |
25 | 52 | ||
26 | - # FOR TEST ONLY | ||
27 | - # Run this agent in simulator mode | ||
28 | - TEST_MODE = False | ||
29 | - # Run the assertion tests at the end | ||
30 | - TEST_WITH_FINAL_TEST = True | ||
31 | - TEST_MAX_DURATION_SEC = None | ||
32 | - #TEST_MAX_DURATION_SEC = 120 | ||
33 | - | ||
34 | - # PM 20190416 fucking config path starting: /home/patrick/Dev/PYROS/start_agent.py agentM | ||
35 | - ##_path_data = 'config' | ||
36 | - _path_data = 'config/old_config' | 53 | + DPRINT = False |
54 | + | ||
55 | + # - status of the sequence after schedule computation | ||
56 | + SEQ_NOT_PROCESSED = 0 | ||
57 | + SEQ_SCHEDULED = 1 | ||
58 | + SEQ_SCHEDULED_OVER_QUOTA = 2 | ||
59 | + SEQ_REJECTED_NO_QUOTA_ENOUGH = -1 | ||
60 | + SEQ_REJECTED_NO_SLOT_AVAILABLE = -2 | ||
61 | + | ||
62 | + # - enum of the matrix line | ||
63 | + SEQ_K = 0 | ||
64 | + SEQ_SEQ_ID = 1 | ||
65 | + SEQ_KOBS0 = 2 | ||
66 | + SEQ_SP_ID = 3 | ||
67 | + SEQ_PRIORITY = 4 | ||
68 | + SEQ_DURATION = 5 | ||
69 | + SEQ_STATUS = 6 | ||
70 | + NB_SEQ = 7 | ||
71 | + | ||
72 | + # - All possible running states | ||
73 | + RUNNING_NOTHING = 0 | ||
74 | + RUNNING_SCHEDULE_PROCESSING = 1 | ||
37 | 75 | ||
38 | - log.debug("PLC instanciated") | 76 | + _AGENT_SPECIFIC_COMMANDS = { |
77 | + # Format : โcmd_nameโ : (timeout, exec_mode) | ||
78 | + "do_compute_schedule_1" : (60, Agent.EXEC_MODE.SEQUENTIAL, ''), | ||
79 | + "do_create_seq_1" : (60, Agent.EXEC_MODE.SEQUENTIAL, ''), | ||
80 | + } | ||
39 | 81 | ||
40 | - _AGENT_SPECIFIC_COMMANDS = [ | ||
41 | - #"do_replan", | ||
42 | - "do_stop_replan", | 82 | + # Scenario to be executed |
83 | + # "self do_stop_current_processing" | ||
84 | + # AgentCmd.CMD_STATUS_CODE.CMD_EXECUTED | ||
85 | + _TEST_COMMANDS_LIST = [ | ||
86 | + # Format : ("self cmd_name cmd_args", timeout, "expected_result", expected_status), | ||
87 | + (True, "self do_create_seq_1 6", 200, '', Agent.CMD_STATUS.CMD_EXECUTED), | ||
88 | + (True, "self do_stop asap", 500, "STOPPING", Agent.CMD_STATUS.CMD_EXECUTED), | ||
43 | ] | 89 | ] |
44 | 90 | ||
45 | - | ||
46 | """ | 91 | """ |
47 | ================================================================= | 92 | ================================================================= |
48 | - FUNCTIONS RUN INSIDE MAIN THREAD | 93 | + Methods running inside main thread |
49 | ================================================================= | 94 | ================================================================= |
50 | """ | 95 | """ |
51 | - # old config | ||
52 | - # @override | ||
53 | - #def __init__(self, name:str=None, config_filename=None, RUN_IN_THREAD=True): | ||
54 | - # def __init__(self, config_filename=None, RUN_IN_THREAD=True): | ||
55 | - # ##if name is None: name = self.__class__.__name__ | ||
56 | - # super().__init__(config_filename, RUN_IN_THREAD) | ||
57 | - | ||
58 | - # new config (obsconfig) | ||
59 | - #def __init__(self, name:str=None, RUN_IN_THREAD=True): | ||
60 | - def __init__(self, name:str=None): | 96 | + def __init__(self, name:str=None,simulated_computer=None): |
61 | if name is None: | 97 | if name is None: |
62 | name = self.__class__.__name__ | 98 | name = self.__class__.__name__ |
63 | - super().__init__() | ||
64 | - #super().__init__(RUN_IN_THREAD) | 99 | + super().__init__(simulated_computer=simulated_computer) |
65 | 100 | ||
66 | - # @override | ||
67 | def _init(self): | 101 | def _init(self): |
68 | super()._init() | 102 | super()._init() |
69 | - log.debug("end init()") | ||
70 | - # --- Set the mode according the startmode value | ||
71 | - ##agent_alias = self.__class__.__name__ | ||
72 | - ##self.set_mode_from_config(agent_alias) | 103 | + log.debug("end super init()") |
104 | + log.info(f"self.TEST_MODE = {self.TEST_MODE}") | ||
73 | 105 | ||
74 | - ''' | ||
75 | - # @override | ||
76 | - def load_config(self): | ||
77 | - super().load_config() | ||
78 | - ''' | 106 | + # === Get the config object |
107 | + self.config = self._oc['config'] | ||
108 | + self.pconfig = self._oc['pyros_config'] | ||
109 | + # === Get agent_alias | ||
110 | + hostname = socket.gethostname() | ||
111 | + log.info(f"{hostname=}") | ||
112 | + log.info(f"{self.name=}") | ||
113 | + agent_alias = self.config.get_agent_real_name(self.name, hostname) | ||
114 | + log.info(f"{agent_alias=}") | ||
79 | 115 | ||
80 | - ''' | ||
81 | - # @override | ||
82 | - def update_survey(self): | ||
83 | - super().update_survey() | ||
84 | - ''' | 116 | + # === Get self._home of current unit |
117 | + self._home = self.config.getHome() | ||
118 | + home = guitastro.Home(self._home) | ||
85 | 119 | ||
86 | - ''' | ||
87 | - # @override | ||
88 | - def get_next_command(self): | ||
89 | - return super().get_next_command() | ||
90 | - ''' | 120 | + self._fn = self.pconfig.fn |
121 | + self._fn.pathnaming("PyROS.seq.1") | ||
91 | 122 | ||
92 | - # @override | ||
93 | - def do_log(self): | ||
94 | - super().do_log() | ||
95 | - | ||
96 | - def replan_sequences(self): | ||
97 | - print("\n start of sequences (re-)planning...\n") | ||
98 | - time.sleep(5) | ||
99 | - sequences = Sequence.objects.filter(status="TBP") | ||
100 | - print("List of sequences to be planned :") | ||
101 | - for seq in sequences: | ||
102 | - print('-', seq.name, '('+seq.status+') :') | ||
103 | - print('-- albums : ', seq.albums.all()) | ||
104 | - print('-- plans : ') | ||
105 | - for album in seq.albums.all(): | ||
106 | - print(album) | ||
107 | - for plan in album.plans.all(): | ||
108 | - print('plan id', plan.id) | ||
109 | - | ||
110 | - print("\n ...end of sequences (re-)planning\n") | 123 | + # === Set longitude to ima object to generate the night yyyymmdd and subdirectories yyyy/mm/dd |
124 | + longitude = home.longitude | ||
125 | + log.info(f"{longitude=}") | ||
126 | + self._fn.longitude(longitude) | ||
127 | + | ||
128 | + # === Status of routine processing | ||
129 | + self._routine_running = self.RUNNING_NOTHING | ||
130 | + log.debug("end init()") | ||
111 | 131 | ||
112 | # Note : called by _routine_process() in Agent | 132 | # Note : called by _routine_process() in Agent |
113 | # @override | 133 | # @override |
114 | def _routine_process_iter_start_body(self): | 134 | def _routine_process_iter_start_body(self): |
115 | - print("The Observatory configuration :") | ||
116 | - self.show_config() | ||
117 | log.debug("in routine_process_before_body()") | 135 | log.debug("in routine_process_before_body()") |
118 | - #self.replan_sequences() | ||
119 | 136 | ||
120 | # Note : called by _routine_process() in Agent | 137 | # Note : called by _routine_process() in Agent |
121 | # @override | 138 | # @override |
122 | def _routine_process_iter_end_body(self): | 139 | def _routine_process_iter_end_body(self): |
123 | - print("The Observatory configuration :") | ||
124 | - #self.show_config() | ||
125 | log.debug("in routine_process_after_body()") | 140 | log.debug("in routine_process_after_body()") |
126 | - self.replan_sequences() | 141 | + # TODO EP est-ce utile ? |
142 | + if self._routine_running == self.RUNNING_NOTHING: | ||
143 | + # Get files to process | ||
144 | + # - Thread TODO | ||
145 | + self._routine_running = self.RUNNING_SCHEDULE_PROCESSING | ||
146 | + self.do_compute_schedule_1() | ||
127 | 147 | ||
128 | - ''' | ||
129 | - # @override | ||
130 | - def exec_specific_cmd_end(self, cmd:Command, from_thread=True): | ||
131 | - super().exec_specific_cmd_end(cmd, from_thread) | ||
132 | - ''' | 148 | + """ |
149 | + ================================================================= | ||
150 | + Methods of specific commands | ||
151 | + ================================================================= | ||
152 | + """ | ||
153 | + | ||
154 | + def do_ccreate_seq_1(self, nb_seq:int): | ||
155 | + """Create sequences to debug | ||
156 | + """ | ||
157 | + self._create_seq_1(nb_seq) | ||
158 | + | ||
159 | + def do_compute_schedule_1(self): | ||
160 | + """Compute a schedule | ||
161 | + | ||
162 | + According the current time, select the night directory. | ||
163 | + List the *.p file list (.p for sequences) | ||
164 | + Read the *.p, *.f file contents (.f for ephemeris) | ||
165 | + Compute the schedule | ||
166 | + | ||
167 | + Output is a matrix to unpack in the database. | ||
168 | + Each line of the matrix is a sequence | ||
169 | + Columns are defined by the enum SEQ_* (see the python code itself). | ||
170 | + | ||
171 | + """ | ||
172 | + self._compute_schedule_1() | ||
173 | + | ||
174 | + """ | ||
175 | + ================================================================= | ||
176 | + Methods called by commands or routine. Overload these methods | ||
177 | + ================================================================= | ||
178 | + # --- | ||
179 | + # osp = ScientificProgram.objects.get(id=scientific_program_id) | ||
180 | + # --- ospperiod is the SP object | ||
181 | + # ospperiod = SP_Period.objects.get(period = period_id, scientific_program = osp) | ||
182 | + # print(f"dir(ospperiod)={dir(ospperiod)}") | ||
183 | + # dir(spperiod)=['DoesNotExist', | ||
184 | + # 'IS_VALID', 'IS_VALID_ACCEPTED', 'IS_VALID_REJECTED', | ||
185 | + # 'MultipleObjectsReturned', 'SP_Period_Guests', 'SP_Period_Users', | ||
186 | + # 'STATUSES', 'STATUSES_ACCEPTED', 'STATUSES_DRAFT', | ||
187 | + # 'STATUSES_EVALUATED', 'STATUSES_REJECTED', 'STATUSES_SUBMITTED', | ||
188 | + # 'VISIBILITY_CHOICES', 'VISIBILITY_NO', 'VISIBILITY_YES', | ||
189 | + # 'VOTES', 'VOTES_NO', 'VOTES_TO_DISCUSS', 'VOTES_YES', | ||
190 | + # 'can_submit_sequence', 'check', 'clean', 'clean_fields', | ||
191 | + # 'date_error_message', 'delete', 'from_db', 'full_clean', | ||
192 | + # 'get_constraints', 'get_deferred_fields', 'get_is_valid_display', | ||
193 | + # 'get_public_visibility_display', 'get_status_display', | ||
194 | + # 'get_vote_referee1_display', 'get_vote_referee2_display', | ||
195 | + # 'id', 'is_currently_active', 'is_valid', 'objects', | ||
196 | + # 'over_quota_duration', 'over_quota_duration_allocated', | ||
197 | + # 'over_quota_duration_remaining', 'period', 'period_id', | ||
198 | + # 'pk', 'prepare_database_save', 'priority', 'public_visibility', | ||
199 | + # 'quota_allocated', 'quota_minimal', 'quota_nominal', | ||
200 | + # 'quota_remaining', 'reason_referee1', 'reason_referee2', | ||
201 | + # 'referee1', 'referee1_id', 'referee2', 'referee2_id', | ||
202 | + # 'refresh_from_db', 'save', 'save_base', 'scientific_program', | ||
203 | + # 'scientific_program_id', 'serializable_value', 'status', 'token', | ||
204 | + # 'token_allocated', 'token_remaining', 'unique_error_message', | ||
205 | + # 'validate_constraints', 'validate_unique', 'vote_referee1', | ||
206 | + # 'vote_referee2' | ||
207 | + """ | ||
208 | + | ||
209 | + def _compute_schedule_1(self): | ||
210 | + """Simple scheduler based on selection-insertion one state algorithm. | ||
211 | + | ||
212 | + Quotas are available only fo the night. | ||
213 | + No token. | ||
214 | + """ | ||
215 | + t0 = time.time() | ||
216 | + #self.DPRINT = True | ||
217 | + # --- Get the incoming directory of the night | ||
218 | + info = self.get_infos() | ||
219 | + rootdir = info['rootdir'] | ||
220 | + subdir = info['subdir'] | ||
221 | + # --- Build the wildcard to list the sequences | ||
222 | + wildcard = os.path.join(rootdir, subdir, "*.p") | ||
223 | + self.dprint(f"{wildcard=}") | ||
224 | + # --- List the sequences from the incoming directory | ||
225 | + seqfiles = glob.glob(wildcard) | ||
226 | + log.info(f"{len(seqfiles)} file sequences to process") | ||
227 | + # --- Initialize the schedule | ||
228 | + schedule = np.zeros(86400, dtype=int) -1 | ||
229 | + schedule_binary = np.ones(86400, dtype=int) | ||
230 | + # =================================================================== | ||
231 | + # --- Loop over the sequences of the night to extract useful infos | ||
232 | + # =================================================================== | ||
233 | + self.dprint("\n" + "="*70 + f"\n=== Read {len(seqfiles)} sequence files of the night {info['night']}\n" + "="*70 + "\n") | ||
234 | + sequence_infos = [] | ||
235 | + # --- Initialize the list of scientific_program_ids | ||
236 | + scientific_program_ids = [] | ||
237 | + kseq = 0 | ||
238 | + for seqfile in seqfiles: | ||
239 | + # --- seqfile = sequence file name | ||
240 | + kseq += 1 | ||
241 | + sequence_info = {} | ||
242 | + sequence_info['id'] = -1 # TBD replace by idseq of the database | ||
243 | + sequence_info['seqfile'] = seqfile | ||
244 | + sequence_info['error'] = "" | ||
245 | + sequence_info['kobs0'] = -1 | ||
246 | + # --- ephfile = ephemeris file name | ||
247 | + ephfile = os.path.splitext(seqfile)[0] + ".f" | ||
248 | + # --- If ephemeris file exists, read files | ||
249 | + if os.path.exists(ephfile): | ||
250 | + self.dprint(f"Read file {seqfile}") | ||
251 | + # --- seq_info = sequence dictionary | ||
252 | + # --- eph_info = ephemeris dictionary | ||
253 | + seq_info = pickle.load(open(seqfile,"rb")) | ||
254 | + #print("="*20 + "\n" + f"{seq_info=}") | ||
255 | + eph_info = pickle.load(open(ephfile,"rb")) | ||
256 | + #print("="*20 + "\n" + f"{eph_info=}") | ||
257 | + # --- | ||
258 | + param = self._fn.naming_get(seqfile) | ||
259 | + sequence_info['id'] = int(param['id_seq']) | ||
260 | + # --- scientific_program_id is an integer | ||
261 | + scientific_program_id = seq_info['sequence']['scientific_program'] | ||
262 | + # --- Dictionary of informations about the sequence | ||
263 | + sequence_info['seq_dico'] = seq_info # useful for duration | ||
264 | + # --- Search the last time when the start of the sequence is observable (visibility > 0) | ||
265 | + visibility_duration = eph_info['visibility_duration'] | ||
266 | + kobss = np.where(visibility_duration > 0) | ||
267 | + kobss = list(kobss[0]) | ||
268 | + if len(kobss) == 0: | ||
269 | + self.dprint(" Sequence has no visibility") | ||
270 | + sequence_info['error'] = f"Sequence has no visibility_duration" | ||
271 | + sequence_infos.append(sequence_info) | ||
272 | + continue | ||
273 | + kobs0 = kobss[0] | ||
274 | + sequence_info['kobs0'] = kobs0 | ||
275 | + sequence_info['visibility'] = eph_info['visibility'] # total slots | ||
276 | + sequence_info['visibility_duration'] = visibility_duration # total slots - duration | ||
277 | + sequence_info['duration'] = seq_info['sequence']['duration'] | ||
278 | + sequence_info['scientific_program_id'] = scientific_program_id | ||
279 | + self.dprint(f" {scientific_program_id=} range to start={len(kobss)}") | ||
280 | + if scientific_program_id not in scientific_program_ids: | ||
281 | + scientific_program_ids.append(scientific_program_id) | ||
282 | + else: | ||
283 | + sequence_info['error'] = f"File {ephfile} not exists" | ||
284 | + sequence_infos.append(sequence_info) | ||
285 | + | ||
286 | + # =================================================================== | ||
287 | + # --- Get informations of priority and quota from scientific programs | ||
288 | + # =================================================================== | ||
289 | + self.dprint("\n" + "="*70 + f"\n=== Get information from {len(scientific_program_ids)} scientific programs of the night\n" + "="*70 + "\n") | ||
290 | + scientific_program_infos = {} | ||
291 | + period_id = info['operiod'].id | ||
292 | + self.dprint(f"{scientific_program_ids=}") | ||
293 | + for scientific_program_id in scientific_program_ids: | ||
294 | + scientific_program_info = {} | ||
295 | + try: | ||
296 | + osp = ScientificProgram.objects.get(id=scientific_program_id) | ||
297 | + # --- ospperiod is the SP object | ||
298 | + ospperiod = SP_Period.objects.get(period = period_id, scientific_program = osp) | ||
299 | + scientific_program_info['priority'] = ospperiod.priority | ||
300 | + scientific_program_info['over_quota_duration'] = ospperiod.over_quota_duration | ||
301 | + scientific_program_info['over_quota_duration_allocated'] = ospperiod.over_quota_duration_allocated | ||
302 | + scientific_program_info['over_quota_duration_remaining'] = ospperiod.over_quota_duration_remaining | ||
303 | + scientific_program_info['quota_allocated'] = ospperiod.quota_allocated | ||
304 | + scientific_program_info['quota_minimal'] = ospperiod.quota_minimal | ||
305 | + scientific_program_info['quota_nominal'] = ospperiod.quota_nominal | ||
306 | + scientific_program_info['quota_remaining'] = ospperiod.quota_remaining | ||
307 | + scientific_program_info['token_allocated'] = ospperiod.token_allocated | ||
308 | + scientific_program_info['token_remaining'] = ospperiod.token_allocated | ||
309 | + except: | ||
310 | + # --- simulation | ||
311 | + scientific_program_info['priority'] = 0 | ||
312 | + if scientific_program_info['priority'] == 0: | ||
313 | + # --- simulation | ||
314 | + priority = 50 + scientific_program_id*5 | ||
315 | + scientific_program_info['priority'] = priority | ||
316 | + scientific_program_info['quota_allocated'] = 12000 | ||
317 | + scientific_program_info['quota_remaining'] = 12000 | ||
318 | + scientific_program_infos[str(scientific_program_id)] = scientific_program_info | ||
319 | + self.dprint(f"{scientific_program_id=} priority={scientific_program_info['priority']} quota={scientific_program_info['quota_remaining']}") | ||
320 | + | ||
321 | + # =================================================================== | ||
322 | + # --- Build the numpy matrix seqs to make rapid computations | ||
323 | + # =================================================================== | ||
324 | + self.dprint("\n" + "="*70 + f"\n=== Build the matrix for scheduling {len(sequence_infos)} sequences\n" + "="*70 + "\n") | ||
325 | + self.dprint("Order ID_seq K_start ID_sp Priority Duration Status\n") | ||
326 | + nseq = len(sequence_infos) | ||
327 | + if nseq == 0: | ||
328 | + self._routine_running = self.RUNNING_NOTHING | ||
329 | + return | ||
330 | + seqs = np.zeros((nseq, self.NB_SEQ), dtype=int) | ||
331 | + k = 0 | ||
332 | + for sequence_info in sequence_infos: | ||
333 | + if 'scientific_program_id' not in sequence_info.keys(): | ||
334 | + self.dprint(f"No scientific program for ID sequence {sequence_info['id']}") | ||
335 | + continue | ||
336 | + scientific_program_id = sequence_info['scientific_program_id'] | ||
337 | + scientific_program_info = scientific_program_infos[str(scientific_program_id)] | ||
338 | + priority = scientific_program_info['priority'] | ||
339 | + # Order of the following list refers to the enum | ||
340 | + seq = [ k, sequence_info['id'], sequence_info['kobs0'], scientific_program_id, priority, int(np.ceil(sequence_info['duration'])), self.SEQ_NOT_PROCESSED ] | ||
341 | + self.dprint(f"{seq=}") | ||
342 | + seqs[k] = seq | ||
343 | + k += 1 | ||
344 | + seqs = seqs[:k] | ||
345 | + # --- Save the matrix sequence | ||
346 | + #print(f"{seqs=}") | ||
347 | + fpathname = os.path.join(rootdir, subdir, "scheduler_seq_matrix1.txt") | ||
348 | + np.savetxt(fpathname, seqs) | ||
349 | + | ||
350 | + # =================================================================== | ||
351 | + # --- Compute the matrix seq_sorteds (priority and chronology) | ||
352 | + # =================================================================== | ||
353 | + self.dprint("\n" + "="*70 + "\n=== Sort the matrix for scheduling by priority and chronology\n" + "="*70 + "\n") | ||
354 | + # --- Sort the matrix sequence: priority=SEQ_PRIORITY (decreasing -1) and then chronology=SEQ_KOBS0 (increasing +1) | ||
355 | + seq_sorteds = seqs[np.lexsort(([1,-1]*seqs[:,[self.SEQ_KOBS0, self.SEQ_PRIORITY]]).T)] | ||
356 | + # --- Save the matrix sequence | ||
357 | + self.dprint("Order ID_seq K_start ID_sp Priority Duration Status\n") | ||
358 | + self.dprint(f"{seq_sorteds=}") | ||
359 | + fpathname = os.path.join(rootdir, subdir, "scheduler_seq_matrix2.txt") | ||
360 | + np.savetxt(fpathname, seq_sorteds) | ||
361 | + | ||
362 | + # =================================================================== | ||
363 | + # --- Insert sequences in the schedule. Respecting priority and quota | ||
364 | + # =================================================================== | ||
365 | + self.dprint("\n" + "="*70 + "\n=== Insertion of the sequences in the schedule respecting priority and quota\n" + "="*70 + "\n") | ||
366 | + for seq in seq_sorteds: | ||
367 | + # --- Unpack the sequence | ||
368 | + k, sequence_id, kobs0, scientific_program_id, priority, duration, seq_status = seq | ||
369 | + | ||
370 | + # --- Get the quota remaining of the scientific program | ||
371 | + quota_remaining = scientific_program_infos[str(scientific_program_id)]['quota_remaining'] | ||
372 | + self.dprint('-'*70 + "\n" + f"Process {sequence_id=} {kobs0=} {duration=} sp_id={scientific_program_id} {quota_remaining=}") | ||
373 | + | ||
374 | + # --- Verify if duration < quota_remaining | ||
375 | + if duration > quota_remaining: | ||
376 | + # --- No remaining quota to insert this sequence | ||
377 | + self.dprint(f"{sequence_id=} cannot be inserted because no quota enough") | ||
378 | + seqs[k][self.SEQ_STATUS] = self.SEQ_REJECTED_NO_QUOTA_ENOUGH | ||
379 | + continue | ||
380 | + | ||
381 | + # --- Compute the remaining visibility and list (k1s) of the best observation start | ||
382 | + # =0 if not possible to start observation | ||
383 | + # =value with the highest value for the best observation start | ||
384 | + | ||
385 | + # --- Visibility*schedule_binary are transformed into binary | ||
386 | + sequence_info = sequence_infos[k] | ||
387 | + vis_binarys = sequence_info['visibility'].copy() * schedule_binary | ||
388 | + vis_binarys[vis_binarys > 0] = 1 | ||
389 | + | ||
390 | + # --- Cumulative sum + offset by -duration to prepare the start_binary computation | ||
391 | + obs_starts = np.cumsum(vis_binarys) | ||
392 | + obs_ends = obs_starts.copy() | ||
393 | + obs_ends[0:-duration] = obs_ends[duration:] | ||
394 | + obs_ends[-duration:] = 0 | ||
395 | + | ||
396 | + # --- Difference and binarisation to get starts with duration | ||
397 | + start_binary = obs_ends - obs_starts | ||
398 | + start_binary[start_binary < duration] = 0 | ||
399 | + start_binary[start_binary == duration] = 1 | ||
400 | + | ||
401 | + # --- Compute the remaining visibility (float) | ||
402 | + remaining_visibility = sequence_info['visibility'] * start_binary | ||
403 | + | ||
404 | + # --- Check the remaining visibility | ||
405 | + if np.sum(remaining_visibility) == 0: | ||
406 | + # --- No remaining slot to insert this sequence | ||
407 | + self.dprint(f"{sequence_id=} cannot inserted because no more slots available") | ||
408 | + seqs[k][self.SEQ_STATUS] = self.SEQ_REJECTED_NO_SLOT_AVAILABLE | ||
409 | + continue | ||
410 | + | ||
411 | + # --- From the index of the highest value of remaining visibility to the index of the lowest value of remaining visibility | ||
412 | + k1s = np.flip(np.argsort(remaining_visibility)) | ||
413 | + self.dprint(f"{k1s=} => Start elevation {sequence_info['visibility'][k1s[0]]:+.2f}") | ||
414 | + | ||
415 | + # --- Get k1 as the highest value of remaining visibility | ||
416 | + k1 = k1s[0] | ||
417 | + k2 = k1 + duration | ||
418 | + self.dprint(f"{k} : {sequence_id=} {scientific_program_id=} {priority=} inserted in the slot {k1=} {k2=} (remaining {quota_remaining - duration} s)") | ||
419 | + | ||
420 | + # --- Update the seqs matrix | ||
421 | + seqs[k][self.SEQ_STATUS] = self.SEQ_SCHEDULED | ||
422 | + | ||
423 | + # --- Update the schedule arrays | ||
424 | + schedule[k1:k2] = sequence_id | ||
425 | + schedule_binary[k1:k2] = 0 | ||
426 | + | ||
427 | + # --- Update the scientific program dict | ||
428 | + quota_remaining -= duration | ||
429 | + scientific_program_infos[str(scientific_program_id)]['quota_remaining'] = quota_remaining | ||
430 | + | ||
431 | + # =================================================================== | ||
432 | + # --- Insert sequences in the schedule. Respecting priority but over quota | ||
433 | + # =================================================================== | ||
434 | + # self.dprint("\n" + "="*70 + "\n=== Insertion of the sequences in the schedule respecting priority but over quota\n" + "="*70 + "\n") | ||
435 | + # TBD | ||
436 | + # where are remaining free slots | ||
437 | + # scan sequences to insert in these free slots | ||
438 | + | ||
439 | + # =================================================================== | ||
440 | + # --- Save the schedule | ||
441 | + # =================================================================== | ||
442 | + self.dprint("\n" + "="*70 + "\n=== Save the schedule\n" + "="*70 + "\n") | ||
443 | + self.dprint("Order ID_seq K_start ID_sp Priority Duration Status\n") | ||
444 | + self.dprint(f"{seqs=}") | ||
445 | + fpathname = os.path.join(rootdir, subdir, "scheduler_schedule.txt") | ||
446 | + np.savetxt(fpathname, np.array([schedule, schedule_binary]).T) | ||
447 | + # --- Update the running state | ||
448 | + self._routine_running = self.RUNNING_NOTHING | ||
449 | + print(f"_compute_schedule_1 finished in {time.time() - t0:.2f} seconds") | ||
133 | 450 | ||
451 | + def _create_seq_1(self, nb_seq): | ||
452 | + t0 = time.time() | ||
453 | + self.dprint("Debut _create_seq_1") | ||
454 | + seq_template = {'sequence': {'id': 4, 'start_expo_pref': 'IMMEDIATE', 'pyros_user': 2, 'scientific_program': 1, 'name': 'seq_20230628T102140', 'desc': None, 'last_modified_by': 2, 'is_alert': False, 'status': 'TBP', 'with_drift': False, 'priority': None, 'analysis_method': None, 'moon_min': None, 'alt_min': None, 'type': None, 'img_current': None, 'img_total': None, 'not_obs': False, 'obsolete': False, 'processing': False, 'flag': None, 'period': 1, 'start_date': datetime.datetime(2023, 6, 28, 10, 21, 40, tzinfo=zoneinfo.ZoneInfo(key='UTC')), 'end_date': datetime.datetime(2023, 6, 28, 10, 21, 40, 999640, tzinfo=datetime.timezone.utc), 'jd1': Decimal('0E-8'), 'jd2': Decimal('0E-8'), 'tolerance_before': '1s', 'tolerance_after': '1min', 'duration': -1.0, 'overhead': Decimal('0E-8'), 'submitted': False, 'config_attributes': {'tolerance_before': '1s', 'tolerance_after': '1min', 'target': 'RADEC 0H10M -15D', 'conformation': 'WIDE', 'layout': 'Altogether'}, 'ra': None, 'dec': None, 'complete': True, 'night_id': '20230627'}, 'albums': {'Altogether': {'plans': [{'id': 4, 'album': 4, 'duration': 0.0, 'nb_images': 1, 'config_attributes': {'binnings': {'binxy': [1, 1], 'readouttime': 6}, 'exposuretime': 1.0}, 'complete': True}]}}} | ||
455 | + info = self.get_infos() | ||
456 | + rootdir = info['rootdir'] | ||
457 | + subdir = info['subdir'] | ||
458 | + # --- Prepare ephemeris object | ||
459 | + # TBD duskelev a parametrer from obsconfig (yml) | ||
460 | + duskelev = -7 | ||
461 | + eph = guitastro.Ephemeris() | ||
462 | + eph.set_home(self.config.getHome()) | ||
463 | + #print("Debut _create_seq_1 SUN") | ||
464 | + ephem_sun = eph.target2night("sun", info['night'], None, None) | ||
465 | + #print("Debut _create_seq_1 MOON") | ||
466 | + ephem_moon = eph.target2night("moon", info['night'], None, None) | ||
467 | + # --- Horizon (TBD get from config) | ||
468 | + #print("Debut _create_seq_1 Horizon") | ||
469 | + hor = guitastro.Horizon(eph.home) | ||
470 | + hor.horizon_altaz = [ [0,0], [360,0] ] | ||
471 | + # --- Delete all existing *.p and *.f files in the night directory | ||
472 | + fn_param = { | ||
473 | + "period" : f"{info['period_id']}", | ||
474 | + "version": "1", | ||
475 | + "unit": self.config.unit_name, | ||
476 | + "date": info['night'], | ||
477 | + "id_seq": 0 | ||
478 | + } | ||
479 | + fname = self._fn.naming_set(fn_param) | ||
480 | + self.dprint(f":: {fname=}") | ||
481 | + seq_file = self._fn.join(fname) | ||
482 | + path_night = os.path.dirname(seq_file) | ||
483 | + cards = ['*.p', '*.f'] | ||
484 | + for card in cards: | ||
485 | + wildcard = os.path.join(path_night, card) | ||
486 | + seq_dfiles = glob.glob(wildcard) | ||
487 | + #print(f"::: {seq_dfiles=}") | ||
488 | + for seq_dfile in seq_dfiles: | ||
489 | + #print(f":::.1 : os.remove {seq_dfile=}") | ||
490 | + os.remove(seq_dfile) | ||
491 | + # --- | ||
492 | + for k in range(nb_seq): | ||
493 | + #print("B"*20 + f" {info['operiod'].id} {info['night']} {k}") | ||
494 | + time.sleep(1) | ||
495 | + seq = seq_template.copy() | ||
496 | + seq['sequence']['period'] = info['operiod'].id # int | ||
497 | + seq['sequence']['night_id'] = info['night'] # str | ||
498 | + seq['sequence']['config_attributes']['target'] = k # int | ||
499 | + # --- | ||
500 | + start_expo_pref = "BESTELEV" #"IMMEDIATE" | ||
501 | + scientific_program = int(k/2) | ||
502 | + start_date = datetime.datetime(2023, 6, 28, 10, 21, 40) | ||
503 | + end_date = datetime.datetime(2023, 6, 28, 10, 21, 40, 999640, tzinfo=datetime.timezone.utc) | ||
504 | + jd1 = Decimal('0E-8') | ||
505 | + jd2 = Decimal('0E-8') | ||
506 | + tolerance_before = '1s' | ||
507 | + tolerance_after = '1min' | ||
508 | + duration = 3000.0 | ||
509 | + target = f"RADEC {k}h {10+2*k}d" | ||
510 | + # --- | ||
511 | + seq['sequence']['start_expo_pref'] = start_expo_pref | ||
512 | + seq['sequence']['scientific_program'] = scientific_program | ||
513 | + seq['sequence']['start_date'] = start_date | ||
514 | + seq['sequence']['end_date'] = end_date | ||
515 | + seq['sequence']['jd1'] = jd1 | ||
516 | + seq['sequence']['jd2'] = jd2 | ||
517 | + seq['sequence']['tolerance_before'] = tolerance_before | ||
518 | + seq['sequence']['tolerance_after'] = tolerance_after | ||
519 | + seq['sequence']['duration'] = duration | ||
520 | + seq['sequence']['config_attributes']['target'] = target | ||
521 | + # --- Build the path and file name of the sequence file | ||
522 | + fn_param["id_seq"] = k | ||
523 | + #print(f"{k} : {self._fn.fcontext=}") | ||
524 | + self._fn.fname = self._fn.naming_set(fn_param) | ||
525 | + #print(f"{k} : {self._fn.fname=}") | ||
526 | + seq_file = self._fn.join(self._fn.fname) | ||
527 | + #print(f"{k} : {seq_file=}") | ||
528 | + # --- Build the path and file name of the ephemeris file | ||
529 | + eph_file = f"{seq_file[:-2]}.f" | ||
530 | + # --- Create directory if it doesn't exist | ||
531 | + #print(f"{k} : {seq_file=}") | ||
532 | + os.makedirs(os.path.dirname(seq_file), exist_ok=True) | ||
533 | + # --- Compute the ephemeris of the sequence and manage errors | ||
534 | + #print(f"{k} : TRY") | ||
535 | + errors = [] | ||
536 | + try: | ||
537 | + # TODO remplacer les none par les fichiers pickle de ephem_sun & ephem_moon | ||
538 | + ephem = eph.target2night(seq["sequence"]["config_attributes"]["target"], info['night'], ephem_sun, ephem_moon, preference=seq['sequence']['start_expo_pref'], duskelev=duskelev, horizon=hor, duration=duration) | ||
539 | + except ValueError: | ||
540 | + errors.append("Target value is not valid") | ||
541 | + except guitastro.ephemeris.EphemerisException as ephemException: | ||
542 | + errors.append(str(ephemException)) | ||
543 | + if len(errors) == 0 and np.sum(ephem["visibility"]) == 0 : | ||
544 | + errors.append("Target is not visible.") | ||
545 | + if len(errors) == 0: | ||
546 | + pickle.dump(ephem, open(eph_file,"wb")) | ||
547 | + pickle.dump(seq, open(seq_file,"wb")) | ||
548 | + #print(f"{errors=}") | ||
549 | + #print("C"*20) | ||
550 | + print(f"_create_seq_1 finished in {time.time() - t0:.2f} seconds") | ||
551 | + | ||
552 | + def load_sequence(self): | ||
553 | + sequence = "" | ||
554 | + return sequence | ||
134 | 555 | ||
556 | + def get_infos(self): | ||
557 | + rootdir = self._fn.rootdir | ||
558 | + operiod = Period.objects.exploitation_period() | ||
559 | + if operiod == None: | ||
560 | + log.info("No period valid in the database") | ||
561 | + self._routine_running = self.RUNNING_NOTHING | ||
562 | + return | ||
563 | + period_id = str(operiod.id) | ||
564 | + if len(str(operiod.id)) < 3: | ||
565 | + while len(period_id) < 3: | ||
566 | + period_id = "0" + period_id | ||
567 | + period_id = "P" + period_id | ||
568 | + night_id = self._fn.date2night("now") | ||
569 | + subdir = os.path.join(period_id, night_id) | ||
570 | + dico = {} | ||
571 | + dico['rootdir'] = rootdir | ||
572 | + dico['subdir'] = subdir | ||
573 | + dico['operiod'] = operiod # object | ||
574 | + dico['period_id'] = period_id # str formated (P000) | ||
575 | + dico['night'] = night_id # str (YYYYMMDD) | ||
576 | + return dico | ||
577 | + | ||
578 | + def dprint(self, *args, **kwargs): | ||
579 | + if self.DPRINT: | ||
580 | + print(*args, **kwargs) | ||
581 | + | ||
135 | if __name__ == "__main__": | 582 | if __name__ == "__main__": |
136 | 583 | ||
137 | - agent = build_agent(A_Scheduler) | 584 | + agent = build_agent(AgentScheduler) |
138 | print(agent) | 585 | print(agent) |
139 | agent.run() | 586 | agent.run() |