Commit b10e2117fc9170a0e3985e0a32c43df7d68af454
Exists in
dev
Merge branch 'dev' of https://gitlab.irap.omp.eu/pyros-irap/pyros into dev
Showing
3 changed files
with
88 additions
and
32 deletions
Show diff stats
src/core/pyros_django/pyros/settings.py
... | ... | @@ -168,7 +168,7 @@ try: |
168 | 168 | |
169 | 169 | except: |
170 | 170 | set_environment_variables_if_not_configured(ENV_PATH,ENV_SAMPLE_PATH) |
171 | -ALLOWED_HOSTS = ['localhost', '127.0.0.1', 'pyros.irap.omp.eu', 'astroguita.hd.free.fr', '0.0.0.0'] | |
171 | +ALLOWED_HOSTS = ['localhost', '127.0.0.1', 'pyros.irap.omp.eu', 'astroguita.freeboxos.fr', '0.0.0.0'] | |
172 | 172 | # defining variables when using Docker |
173 | 173 | if WITH_DOCKER: |
174 | 174 | ALLOWED_HOSTS.append('0.0.0.0') | ... | ... |
src/core/pyros_django/scheduling/A_Scheduler.py
... | ... | @@ -32,13 +32,14 @@ if pwd not in sys.path: |
32 | 32 | |
33 | 33 | short_paths = ['src', 'src/core/pyros_django'] |
34 | 34 | for short_path in short_paths: |
35 | - path = os.path.join(pwd, short_path) | |
35 | + path = os.path.abspath(os.path.join(pwd, short_path)) | |
36 | 36 | if path not in sys.path: |
37 | 37 | sys.path.insert(0, path) |
38 | 38 | |
39 | 39 | from src.core.pyros_django.majordome.agent.Agent import Agent, build_agent, log, parse_args |
40 | 40 | from seq_submit.models import Sequence |
41 | 41 | from user_mgmt.models import Period, ScientificProgram, SP_Period |
42 | +from scheduling.models import PredictiveSchedule, EffectiveSchedule | |
42 | 43 | # = Specials |
43 | 44 | import glob |
44 | 45 | import shutil |
... | ... | @@ -85,7 +86,7 @@ class A_Scheduler(Agent): |
85 | 86 | _TEST_COMMANDS_LIST = [ |
86 | 87 | # Format : ("self cmd_name cmd_args", timeout, "expected_result", expected_status), |
87 | 88 | (True, "self do_create_seq_1 6", 200, '', Agent.CMD_STATUS.CMD_EXECUTED), |
88 | - (True, "self do_stop asap", 500, "STOPPING", Agent.CMD_STATUS.CMD_EXECUTED), | |
89 | + #(True, "self do_stop asap", 500, "STOPPING", Agent.CMD_STATUS.CMD_EXECUTED), | |
89 | 90 | ] |
90 | 91 | |
91 | 92 | """ |
... | ... | @@ -128,6 +129,7 @@ class A_Scheduler(Agent): |
128 | 129 | # === Status of routine processing |
129 | 130 | self._routine_running = self.RUNNING_NOTHING |
130 | 131 | log.debug("end init()") |
132 | + ##### TBD suppress redondant paths in print(f"=>=>=> {sys.path=}") | |
131 | 133 | |
132 | 134 | # Note : called by _routine_process() in Agent |
133 | 135 | # @override |
... | ... | @@ -151,7 +153,7 @@ class A_Scheduler(Agent): |
151 | 153 | ================================================================= |
152 | 154 | """ |
153 | 155 | |
154 | - def do_ccreate_seq_1(self, nb_seq:int): | |
156 | + def do_create_seq_1(self, nb_seq:int): | |
155 | 157 | """Create sequences to debug |
156 | 158 | """ |
157 | 159 | self._create_seq_1(nb_seq) |
... | ... | @@ -213,7 +215,7 @@ class A_Scheduler(Agent): |
213 | 215 | No token. |
214 | 216 | """ |
215 | 217 | t0 = time.time() |
216 | - #self.DPRINT = True | |
218 | + self.DPRINT = True | |
217 | 219 | # --- Get the incoming directory of the night |
218 | 220 | info = self.get_infos() |
219 | 221 | rootdir = info['rootdir'] |
... | ... | @@ -225,8 +227,30 @@ class A_Scheduler(Agent): |
225 | 227 | seqfiles = glob.glob(wildcard) |
226 | 228 | log.info(f"{len(seqfiles)} file sequences to process") |
227 | 229 | # --- Initialize the schedule |
228 | - schedule = np.zeros(86400, dtype=int) -1 | |
230 | + schedule_sequence_id = np.zeros(86400, dtype=int) -1 | |
229 | 231 | schedule_binary = np.ones(86400, dtype=int) |
232 | + schedule_visibility = np.zeros(86400, dtype=float) | |
233 | + schedule_order = np.zeros(86400, dtype=int) -1 | |
234 | + schedule_jd = np.zeros(86400, dtype=float) | |
235 | + schedule_scientific_programm_id = np.zeros(86400, dtype=int) -1 | |
236 | + # --- Get the numpy matrix in database (via Json) | |
237 | + try: | |
238 | + # --- Get the effective schedule matrix | |
239 | + input_matrix = EffectiveSchedule.objects.last().conv_numpy() | |
240 | + # --- Get the effective schedule arrays | |
241 | + schedule_eff_jd, schedule_eff_binary, schedule_eff_sequence_id, schedule_eff_scientific_programm_id, schedule_eff_order, schedule_eff_visibility = input_matrix | |
242 | + # --- Get the index in the night | |
243 | + night, index = self._fn.date2night("now", 86400) | |
244 | + schedule_sequence_id[0:index] = schedule_eff_sequence_id[0:index] | |
245 | + schedule_binary[0:index] = schedule_eff_binary[0:index] | |
246 | + schedule_visibility[0:index] = schedule_eff_visibility[0:index] | |
247 | + schedule_order[0:index] = schedule_eff_order[0:index] | |
248 | + schedule_jd[0:index] = schedule_eff_jd[0:index] | |
249 | + schedule_scientific_programm_id[0:index] = schedule_eff_scientific_programm_id[0:index] | |
250 | + except: | |
251 | + pass | |
252 | + #print(f"{schedule_jd=}") | |
253 | + | |
230 | 254 | # =================================================================== |
231 | 255 | # --- Loop over the sequences of the night to extract useful infos |
232 | 256 | # =================================================================== |
... | ... | @@ -282,6 +306,10 @@ class A_Scheduler(Agent): |
282 | 306 | else: |
283 | 307 | sequence_info['error'] = f"File {ephfile} not exists" |
284 | 308 | sequence_infos.append(sequence_info) |
309 | + try: | |
310 | + schedule_jd = eph_info['jd'] | |
311 | + except: | |
312 | + pass | |
285 | 313 | |
286 | 314 | # =================================================================== |
287 | 315 | # --- Get informations of priority and quota from scientific programs |
... | ... | @@ -337,7 +365,7 @@ class A_Scheduler(Agent): |
337 | 365 | scientific_program_info = scientific_program_infos[str(scientific_program_id)] |
338 | 366 | priority = scientific_program_info['priority'] |
339 | 367 | # Order of the following list refers to the enum |
340 | - seq = [ k, sequence_info['id'], sequence_info['kobs0'], scientific_program_id, priority, int(np.ceil(sequence_info['duration'])), self.SEQ_NOT_PROCESSED ] | |
368 | + seq = [ k, sequence_info['id'], sequence_info['kobs0'], scientific_program_id, priority, int(np.ceil(sequence_info['duration'])), self.SEQ_NOT_PROCESSED] | |
341 | 369 | self.dprint(f"{seq=}") |
342 | 370 | seqs[k] = seq |
343 | 371 | k += 1 |
... | ... | @@ -363,9 +391,12 @@ class A_Scheduler(Agent): |
363 | 391 | # --- Insert sequences in the schedule. Respecting priority and quota |
364 | 392 | # =================================================================== |
365 | 393 | self.dprint("\n" + "="*70 + "\n=== Insertion of the sequences in the schedule respecting priority and quota\n" + "="*70 + "\n") |
366 | - for seq in seq_sorteds: | |
394 | + kseq_sorted = -1 | |
395 | + for seq_sorted in seq_sorteds: | |
396 | + kseq_sorted += 1 | |
397 | + | |
367 | 398 | # --- Unpack the sequence |
368 | - k, sequence_id, kobs0, scientific_program_id, priority, duration, seq_status = seq | |
399 | + k, sequence_id, kobs0, scientific_program_id, priority, duration, seq_status = seq_sorted | |
369 | 400 | |
370 | 401 | # --- Get the quota remaining of the scientific program |
371 | 402 | quota_remaining = scientific_program_infos[str(scientific_program_id)]['quota_remaining'] |
... | ... | @@ -421,12 +452,16 @@ class A_Scheduler(Agent): |
421 | 452 | seqs[k][self.SEQ_STATUS] = self.SEQ_SCHEDULED |
422 | 453 | |
423 | 454 | # --- Update the schedule arrays |
424 | - schedule[k1:k2] = sequence_id | |
455 | + schedule_sequence_id[k1:k2] = sequence_id | |
425 | 456 | schedule_binary[k1:k2] = 0 |
457 | + schedule_visibility[k1:k2] = sequence_info['visibility'][k1:k2] | |
458 | + schedule_order[k1:k2] = kseq_sorted | |
459 | + schedule_scientific_programm_id[k1:k2] = scientific_program_id | |
426 | 460 | |
427 | 461 | # --- Update the scientific program dict |
428 | 462 | quota_remaining -= duration |
429 | 463 | scientific_program_infos[str(scientific_program_id)]['quota_remaining'] = quota_remaining |
464 | + | |
430 | 465 | |
431 | 466 | # =================================================================== |
432 | 467 | # --- Insert sequences in the schedule. Respecting priority but over quota |
... | ... | @@ -442,8 +477,25 @@ class A_Scheduler(Agent): |
442 | 477 | self.dprint("\n" + "="*70 + "\n=== Save the schedule\n" + "="*70 + "\n") |
443 | 478 | self.dprint("Order ID_seq K_start ID_sp Priority Duration Status\n") |
444 | 479 | self.dprint(f"{seqs=}") |
480 | + # --- Prepare the output matrix | |
481 | + ouput_matrix = np.array([schedule_jd, schedule_binary, schedule_sequence_id, schedule_scientific_programm_id, schedule_order, schedule_visibility]) | |
482 | + # --- Save the numpy matrix in ASCII | |
445 | 483 | fpathname = os.path.join(rootdir, subdir, "scheduler_schedule.txt") |
446 | - np.savetxt(fpathname, np.array([schedule, schedule_binary]).T) | |
484 | + np.savetxt(fpathname, ouput_matrix.T) | |
485 | + # --- Save the numpy matrix in database (via Json) | |
486 | + try: | |
487 | + v = PredictiveSchedule.objects.last() | |
488 | + except: | |
489 | + v = PredictiveSchedule() | |
490 | + v.scheduler_matrix = ouput_matrix | |
491 | + v.save() | |
492 | + # --- Save the numpy matrix in database (via Json) | |
493 | + try: | |
494 | + v = EffectiveSchedule.objects.last() | |
495 | + except: | |
496 | + v = EffectiveSchedule() | |
497 | + v.scheduler_matrix = ouput_matrix | |
498 | + v.save() | |
447 | 499 | # --- Update the running state |
448 | 500 | self._routine_running = self.RUNNING_NOTHING |
449 | 501 | print(f"_compute_schedule_1 finished in {time.time() - t0:.2f} seconds") |
... | ... | @@ -452,6 +504,7 @@ class A_Scheduler(Agent): |
452 | 504 | t0 = time.time() |
453 | 505 | self.dprint("Debut _create_seq_1") |
454 | 506 | seq_template = {'sequence': {'id': 4, 'start_expo_pref': 'IMMEDIATE', 'pyros_user': 2, 'scientific_program': 1, 'name': 'seq_20230628T102140', 'desc': None, 'last_modified_by': 2, 'is_alert': False, 'status': 'TBP', 'with_drift': False, 'priority': None, 'analysis_method': None, 'moon_min': None, 'alt_min': None, 'type': None, 'img_current': None, 'img_total': None, 'not_obs': False, 'obsolete': False, 'processing': False, 'flag': None, 'period': 1, 'start_date': datetime.datetime(2023, 6, 28, 10, 21, 40, tzinfo=zoneinfo.ZoneInfo(key='UTC')), 'end_date': datetime.datetime(2023, 6, 28, 10, 21, 40, 999640, tzinfo=datetime.timezone.utc), 'jd1': Decimal('0E-8'), 'jd2': Decimal('0E-8'), 'tolerance_before': '1s', 'tolerance_after': '1min', 'duration': -1.0, 'overhead': Decimal('0E-8'), 'submitted': False, 'config_attributes': {'tolerance_before': '1s', 'tolerance_after': '1min', 'target': 'RADEC 0H10M -15D', 'conformation': 'WIDE', 'layout': 'Altogether'}, 'ra': None, 'dec': None, 'complete': True, 'night_id': '20230627'}, 'albums': {'Altogether': {'plans': [{'id': 4, 'album': 4, 'duration': 0.0, 'nb_images': 1, 'config_attributes': {'binnings': {'binxy': [1, 1], 'readouttime': 6}, 'exposuretime': 1.0}, 'complete': True}]}}} |
507 | + # decode general variables info a dict info | |
455 | 508 | info = self.get_infos() |
456 | 509 | rootdir = info['rootdir'] |
457 | 510 | subdir = info['subdir'] |
... | ... | @@ -460,12 +513,15 @@ class A_Scheduler(Agent): |
460 | 513 | duskelev = -7 |
461 | 514 | eph = guitastro.Ephemeris() |
462 | 515 | eph.set_home(self.config.getHome()) |
463 | - #print("Debut _create_seq_1 SUN") | |
464 | - ephem_sun = eph.target2night("sun", info['night'], None, None) | |
465 | - #print("Debut _create_seq_1 MOON") | |
516 | + print(f"Debut _create_seq_1 SUN {info['night']=}") | |
517 | + try: | |
518 | + ephem_sun = eph.target2night("sun", info['night'], None, None) | |
519 | + except Exception as e: | |
520 | + print(f"{e.message=} {e.args=}") | |
521 | + print(f"Debut _create_seq_1 MOON {eph.target2night=}") | |
466 | 522 | ephem_moon = eph.target2night("moon", info['night'], None, None) |
467 | 523 | # --- Horizon (TBD get from config) |
468 | - #print("Debut _create_seq_1 Horizon") | |
524 | + print("Debut _create_seq_1 Horizon") | |
469 | 525 | hor = guitastro.Horizon(eph.home) |
470 | 526 | hor.horizon_altaz = [ [0,0], [360,0] ] |
471 | 527 | # --- Delete all existing *.p and *.f files in the night directory |
... | ... | @@ -520,15 +576,15 @@ class A_Scheduler(Agent): |
520 | 576 | seq['sequence']['config_attributes']['target'] = target |
521 | 577 | # --- Build the path and file name of the sequence file |
522 | 578 | fn_param["id_seq"] = k |
523 | - #print(f"{k} : {self._fn.fcontext=}") | |
579 | + print(f"{k} : {self._fn.fcontext=}") | |
524 | 580 | self._fn.fname = self._fn.naming_set(fn_param) |
525 | - #print(f"{k} : {self._fn.fname=}") | |
581 | + print(f"{k} : {self._fn.fname=}") | |
526 | 582 | seq_file = self._fn.join(self._fn.fname) |
527 | - #print(f"{k} : {seq_file=}") | |
583 | + print(f"{k} : {seq_file=}") | |
528 | 584 | # --- Build the path and file name of the ephemeris file |
529 | 585 | eph_file = f"{seq_file[:-2]}.f" |
530 | 586 | # --- Create directory if it doesn't exist |
531 | - #print(f"{k} : {seq_file=}") | |
587 | + print(f"{k} : {seq_file=}") | |
532 | 588 | os.makedirs(os.path.dirname(seq_file), exist_ok=True) |
533 | 589 | # --- Compute the ephemeris of the sequence and manage errors |
534 | 590 | #print(f"{k} : TRY") | ... | ... |
src/core/pyros_django/scheduling/models.py
... | ... | @@ -7,6 +7,7 @@ import numpy as np |
7 | 7 | import json |
8 | 8 | from json import JSONEncoder |
9 | 9 | import numpy |
10 | +import ast | |
10 | 11 | |
11 | 12 | class NumpyArrayEncoder(JSONEncoder): |
12 | 13 | def default(self, obj): |
... | ... | @@ -15,40 +16,39 @@ class NumpyArrayEncoder(JSONEncoder): |
15 | 16 | return JSONEncoder.default(self, obj) |
16 | 17 | |
17 | 18 | |
19 | + | |
18 | 20 | |
19 | 21 | class EffectiveSchedule(models.Model): |
20 | 22 | scheduler_matrix = JSONField(blank=True, null=True) |
21 | - | |
23 | + | |
24 | + | |
22 | 25 | def save(self, *args, **kwargs): |
23 | 26 | # Transform numpy matrix to JSON |
24 | 27 | scheduler_matrix_as_json = json.dumps(self.scheduler_matrix, cls=NumpyArrayEncoder) |
25 | 28 | self.scheduler_matrix = scheduler_matrix_as_json |
26 | 29 | super(EffectiveSchedule, self).save(*args, **kwargs) |
27 | 30 | |
28 | - def get(self, *args, **kwargs): | |
31 | + def conv_numpy(self): | |
29 | 32 | # Transform JSON to numpy matrix |
30 | 33 | matrix_as_json = json.loads(self.scheduler_matrix) |
31 | - | |
32 | - self.scheduler_matrix = numpy.asarray(matrix_as_json["array"]) | |
33 | - return super().get(**kwargs) | |
34 | - | |
34 | + | |
35 | + return numpy.array(matrix_as_json) | |
36 | + | |
35 | 37 | class PredictiveSchedule(models.Model): |
36 | 38 | scheduler_matrix = JSONField(blank=True, null=True) |
37 | - | |
39 | + | |
38 | 40 | def save(self, *args, **kwargs): |
39 | 41 | # Transform numpy matrix to JSON |
40 | 42 | scheduler_matrix_as_json = json.dumps(self.scheduler_matrix, cls=NumpyArrayEncoder) |
41 | 43 | self.scheduler_matrix = scheduler_matrix_as_json |
42 | 44 | super(PredictiveSchedule, self).save(*args, **kwargs) |
43 | 45 | |
44 | - def get(self, *args, **kwargs): | |
46 | + def conv_numpy(self, *args, **kwargs): | |
45 | 47 | # Transform JSON to numpy matrix |
46 | 48 | matrix_as_json = json.loads(self.scheduler_matrix) |
47 | - | |
48 | - self.scheduler_matrix = numpy.asarray(matrix_as_json["array"]) | |
49 | - return super().get(**kwargs) | |
50 | - | |
51 | - | |
49 | + | |
50 | + return numpy.array(matrix_as_json) | |
51 | + | |
52 | 52 | class SchedulerHistory(EffectiveSchedule): |
53 | 53 | night_datetime = models.DateTimeField(blank=True, null=True) |
54 | 54 | |
55 | 55 | \ No newline at end of file | ... | ... |