From 673ff0981498d303c3b0611d2bf7df5434585dd3 Mon Sep 17 00:00:00 2001 From: pyros_astroguita Date: Fri, 20 Oct 2023 16:55:59 +0200 Subject: [PATCH] Quota progression (with sequence, quota sp now attached to SP_Period) --- src/core/pyros_django/majordome/agent/Agent.py | 50 +++++++++++++++++++++++++++++++++++++++++++++----- src/core/pyros_django/scheduling/A_Scheduler.py | 95 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------------- src/core/pyros_django/scp_mgmt/A_SCP_Manager.py | 6 +++--- src/core/pyros_django/scp_mgmt/models.py | 6 ++++++ src/core/pyros_django/seq_submit/views.py | 2 +- src/core/pyros_django/user_mgmt/models.py | 10 +++++----- 6 files changed, 129 insertions(+), 40 deletions(-) diff --git a/src/core/pyros_django/majordome/agent/Agent.py b/src/core/pyros_django/majordome/agent/Agent.py index 01314ce..21d920f 100755 --- a/src/core/pyros_django/majordome/agent/Agent.py +++ b/src/core/pyros_django/majordome/agent/Agent.py @@ -154,7 +154,7 @@ import config.old_config as config_old #from config import * from majordome.models import AgentSurvey, AgentCmd, AgentLogs -from user_mgmt.models import Period +from user_mgmt.models import Period, Institute, ScientificProgram, SP_Period from scp_mgmt.models import Quota from vendor.guitastro.src.guitastro import Ephemeris @@ -3576,8 +3576,10 @@ class Agent: d_prev = 0 d_cur = 0 d_total = 0 + dusk_elev1, dusk_elev2 = self.config.getDuskElev() + dawn_elev1, dawn_elev2 = self.config.getDawnElev() night_info = {} - self._duskelev = self.config.getDuskElev() + _ ,self._duskelev = self.config.getDuskElev() while jd < jd2: night = self._oc['config'].fn.date2night(jd) for target in targets: @@ -3586,13 +3588,15 @@ class Agent: ks = np.where(ephem['alt'] < self._duskelev) d_prev = d_prev + d_cur d_cur = len(ks[0]) - night_info[night] = [0, d_prev, d_cur, 0] + _, start_index = self.get_indexes_of_night(ephem, dusk_elev1, dusk_elev2) + night_info[night] = [0, d_prev, d_cur, 0, start_index] + jd += 1 d_total = d_prev + d_cur for key, val in night_info.items(): night_id = key val[0] = d_total - val[-1] = d_total - val[-2] + val[-2] = d_total - val[-3] # --- update db TODO try: if Quota.objects.get(night_id=night_id): @@ -3607,7 +3611,8 @@ class Agent: quota_attributes["d_total"] = val[0] quota_attributes["d_previous"] = val[1] quota_attributes["d_current"] = val[2] - quota_attributes["d_next"] = val[-1] + quota_attributes["d_next"] = val[3] + quota_attributes["index_start"] = val[4] quota.set_attributes_and_save(quota_attributes) operiod.quota = Quota.objects.get(night_id=list(night_info.keys())[0], id_period=operiod.id) operiod.save() @@ -3615,6 +3620,41 @@ class Agent: #pickle.dump(night_info, open(filename, "wb")) return night_info + def update_quota_institutes_and_sp_for_night(self, quota_night): + for institute in Institute.objects.all(): + institute_new_quota = quota_night.convert_to_quota(institute.quota_f) + institute.quota.set_attributes_and_save(institute_new_quota) + for sp_period in SP_Period.objects.filter(scientific_program__in=institute.scientific_programs.all(), period=Period.objects.get(id=quota_night.id_period)): + sp_new_quota = institute.quota.convert_to_quota(sp_period.quota_f) + + sp_period.quota.set_attributes_and_save(sp_new_quota) + + def get_indexes_of_night(self, ephemeris:dict, dusk_elev1:int, dusk_elev2:int)->Tuple: + es = ephemeris["alt"] + delta_w = dusk_elev2 - dusk_elev1 + delta_es = es[1:] - es[:-1] + delta_es = np.append(delta_es, delta_es[-1]) + ks = np.linspace(0, 86400, 86401) + k1 = -1 + k2 = -1 + k = 0 + if delta_w > 0: + for e, delta_e, k in zip(es, delta_es, ks): + if e > dusk_elev1 and k1 == -1 and delta_e > 0: + k1 = k + elif e > dusk_elev2 and k2 == -1 and delta_e > 0: + k2=k + break + else: + for e, delta_e, k in zip(es, delta_es, ks): + + if e < dusk_elev1 and k1 == -1 and delta_e < 0: + k1=k + elif e < dusk_elev2 and k2 == -1 and delta_e < 0: + k2=k + break + + return (k1, k2) """ ================================================================= MAIN diff --git a/src/core/pyros_django/scheduling/A_Scheduler.py b/src/core/pyros_django/scheduling/A_Scheduler.py index 0490688..d860238 100755 --- a/src/core/pyros_django/scheduling/A_Scheduler.py +++ b/src/core/pyros_django/scheduling/A_Scheduler.py @@ -232,6 +232,7 @@ class A_Scheduler(Agent): def update_db_quota_sequence(self, sequence_id, quota_attributes): sequence = Sequence.objects.get(id=sequence_id) + new_quota = Quota() new_quota.set_attributes_and_save(quota_attributes) sequence.quota = new_quota @@ -291,7 +292,21 @@ class A_Scheduler(Agent): # --- Unpack the matrix to effective schedule arrays schedule_eff_jd, schedule_eff_binary, schedule_eff_sequence_id, schedule_eff_scientific_programm_id, schedule_eff_order, schedule_eff_visibility = input_matrix # --- Get the index of the current instant in the night - nownight, index = self._fn.date2night("now", self.BINS_NIGHT) + date = guitastro.Date("now") + "12H" + nownight, index = self._fn.date2night(date.iso(), self.BINS_NIGHT) + index_stop = oquota.index_start + oquota.d_current + #print(f"{index=} {index_stop=} {oquota.index_start=}") + if index > index_stop: + period_quota_passed = oquota.d_current + elif index > oquota.index_start: + period_quota_passed = index - oquota.index_start + else: + period_quota_passed = 0 + oquota.d_passed = period_quota_passed + period_quota_to_schedule = oquota.d_current - period_quota_passed + oquota.d_schedule = period_quota_to_schedule + oquota.save() + self.update_quota_institutes_and_sp_for_night(oquota) self.dprint(f"{nownight=} {index=}") # --- Add all ever observed sequences from 0 to index if nownight == night and (index >= 0 or index < self.BINS_NIGHT): @@ -309,6 +324,24 @@ class A_Scheduler(Agent): print(f"Invalid entry in the database") #print(f"{schedule_jd=}") + # Set global quota for SP and institutes for current night + date = guitastro.Date("now") + "240H" + nownight, index = self._fn.date2night(date.iso(), self.BINS_NIGHT) + index_stop = oquota.index_start + oquota.d_current + #print(f"{index=} {index_stop=} {oquota.index_start=}") + if index > index_stop: + period_quota_passed = oquota.d_current + elif index > oquota.index_start: + period_quota_passed = index - oquota.index_start + else: + period_quota_passed = 0 + oquota.d_passed = period_quota_passed + period_quota_to_schedule = oquota.d_current - period_quota_passed + oquota.d_schedule = period_quota_to_schedule + oquota.save() + self.update_quota_institutes_and_sp_for_night(oquota) + + # =================================================================== # --- Loop over the sequences of the night to extract useful infos # =================================================================== @@ -389,29 +422,18 @@ class A_Scheduler(Agent): self.dprint(f"{scientific_program_ids=}") for scientific_program_id in scientific_program_ids: scientific_program_info = {} - try: - osp = ScientificProgram.objects.get(id=scientific_program_id) - # --- ospperiod is the SP object - ospperiod = SP_Period.objects.get(period = period_id, scientific_program = osp) - scientific_program_info['priority'] = ospperiod.priority - scientific_program_info['over_quota_duration'] = ospperiod.over_quota_duration - scientific_program_info['over_quota_duration_allocated'] = ospperiod.over_quota_duration_allocated - scientific_program_info['over_quota_duration_remaining'] = ospperiod.over_quota_duration_remaining - scientific_program_info['quota_allocated'] = ospperiod.quota_allocated - scientific_program_info['quota_minimal'] = ospperiod.quota_minimal - scientific_program_info['quota_nominal'] = ospperiod.quota_nominal - scientific_program_info['quota_remaining'] = ospperiod.quota_remaining - scientific_program_info['token_allocated'] = ospperiod.token_allocated - scientific_program_info['token_remaining'] = ospperiod.token_allocated - except: - # --- simulation - scientific_program_info['priority'] = 0 - if scientific_program_info['priority'] == 0: - # --- simulation - priority = 50 + scientific_program_id*5 - scientific_program_info['priority'] = priority - scientific_program_info['quota_allocated'] = 12000 - scientific_program_info['quota_remaining'] = 12000 + ospperiod = SP_Period.objects.get(scientific_program=ScientificProgram.objects.get(id=scientific_program_id), period=info["operiod"]) + # --- ospperiod is the SP object + scientific_program_info['priority'] =ospperiod.priority + quota_ospperiod = ospperiod.quota + # example to get d_current of sp_period + # quota_ospperiod.d_current + scientific_program_info['token_allocated'] = ospperiod.token_allocated + scientific_program_info['token_remaining'] = ospperiod.token_allocated + scientific_program_info['quota_allocated'] = quota_ospperiod.d_current + scientific_program_info['quota_remaining'] = quota_ospperiod.d_schedule + scientific_program_info['d_currentq'] = 0 + scientific_program_infos[str(scientific_program_id)] = scientific_program_info self.dprint(f"{scientific_program_id=} priority={scientific_program_info['priority']} quota={scientific_program_info['quota_remaining']}") @@ -530,7 +552,28 @@ class A_Scheduler(Agent): # --- Update the scientific program dict quota_remaining -= duration scientific_program_infos[str(scientific_program_id)]['quota_remaining'] = quota_remaining - + + # --- Update the DB + osequence = Sequence.objects.get(id=sequence_id) + sequence_quota = osequence.quota + sequence_quota.d_schedule = 0 + sequence_quota.d_currentq = duration + sequence_quota.d_scheduleq = duration + osequence.quota.save() + # quota_ospperiod.d_currentq += duration + # quota_ospperiod.save() + scientific_program_infos[str(scientific_program_id)]['d_currentq'] += duration + + for scientific_program_id in scientific_program_ids: + ospperiod = SP_Period.objects.get(scientific_program=ScientificProgram.objects.get(id=scientific_program_id), period=info["operiod"]) + quota_ospperiod = ospperiod.quota + quota_ospperiod.d_currentq = scientific_program_infos[str(scientific_program_id)]['d_currentq'] + quota_ospperiod.d_scheduleq = scientific_program_infos[str(scientific_program_id)]['d_currentq'] + ospperiod.quota.save() + # for sp in SP_Period.objects.filter(period=Period.objects.get(id=period_id)): + # institute_quota = sp.scientific_program.institute.quota + # institute_quota.d_currentq += sp.quota.d_currentq + # sp.scientific_program.institute.save() # =================================================================== # --- Insert sequences in the schedule. Respecting priority but over quota @@ -632,7 +675,7 @@ class A_Scheduler(Agent): #start_expo_pref = "BESTELEV" #"IMMEDIATE" start_expo_pref = 0 # for bestelev 1, for immediate 0 start_date,_ = self._fn.night2date(info["night"]) - start_date = start_date + 0.25 + (0.5*k) + start_date = start_date + 0.25 + (0.05*k) start_date_to_datetime = guitastro.Date(start_date).iso() #start_date = datetime.datetime(2023, 6, 28, 10, 21, 40) #end_date = datetime.datetime(2023, 6, 28, 10, 21, 40, 999640, tzinfo=datetime.timezone.utc) diff --git a/src/core/pyros_django/scp_mgmt/A_SCP_Manager.py b/src/core/pyros_django/scp_mgmt/A_SCP_Manager.py index 90b65c6..9749232 100644 --- a/src/core/pyros_django/scp_mgmt/A_SCP_Manager.py +++ b/src/core/pyros_django/scp_mgmt/A_SCP_Manager.py @@ -287,12 +287,12 @@ class A_SCP_Manager(Agent): institute = sp.institute institute_quota = institute.quota new_quota = Quota() - quota_attributes = institute_quota.convert_to_quota(sp.quota_f) + quota_attributes = institute_quota.convert_to_quota(sp_period.quota_f) quota_attributes["night_id"] = 1 quota_attributes["id_period"] = id_period new_quota.set_attributes_and_save(quota_attributes) - sp.quota = new_quota - sp.save() + sp_period.quota = new_quota + sp_period.save() except Exception as e: print(e) diff --git a/src/core/pyros_django/scp_mgmt/models.py b/src/core/pyros_django/scp_mgmt/models.py index 6973886..98cc42a 100644 --- a/src/core/pyros_django/scp_mgmt/models.py +++ b/src/core/pyros_django/scp_mgmt/models.py @@ -4,6 +4,9 @@ from django.db import models class Quota(models.Model): id_period = models.BigIntegerField(blank=True, null=True) night_id = models.CharField(max_length=8, null=True, blank=True, db_index=True) + + # in seconds + index_start = models.BigIntegerField(default=0, blank=True, null=True) d_total = models.BigIntegerField(default=0, blank=True, null=True) d_totalq = models.BigIntegerField(default=0, blank=True, null=True) d_totalx = models.BigIntegerField(default=0, blank=True, null=True) @@ -57,6 +60,9 @@ class Quota(models.Model): self.id_period = quota_attributes["id_period"] if quota_attributes.get("night_id") != None: self.night_id = quota_attributes["night_id"] + + if quota_attributes.get("index_start") != None: + self.index_start = quota_attributes.get("index_start") if quota_attributes.get("d_totalq") != None: self.d_totalq = quota_attributes["d_totalq"] diff --git a/src/core/pyros_django/seq_submit/views.py b/src/core/pyros_django/seq_submit/views.py index d17347f..ebb4559 100644 --- a/src/core/pyros_django/seq_submit/views.py +++ b/src/core/pyros_django/seq_submit/views.py @@ -305,7 +305,7 @@ def sequence_validate(request, seq_id): eph = guitastro.Ephemeris() eph.set_home(home) # duskelev a parametrer dans obsconfig (yml) - duskelev = -7 + _, duskelev = config.getDuskElev() message = "" try: # TODO remplacer les none par les fichiers pickle de ephem_sun & ephem_moon diff --git a/src/core/pyros_django/user_mgmt/models.py b/src/core/pyros_django/user_mgmt/models.py index 872fa81..e6eadca 100644 --- a/src/core/pyros_django/user_mgmt/models.py +++ b/src/core/pyros_django/user_mgmt/models.py @@ -70,7 +70,6 @@ class UserLevel(models.Model): class Country(models.Model): name = models.CharField(max_length=45, blank=True, null=True) desc = models.TextField(blank=True, null=True) - quota = models.FloatField(blank=True, null=True) class Meta: managed = True @@ -86,7 +85,7 @@ class Institute(models.Model): # fraction quota quota_f = models.FloatField( validators=[MinValueValidator(0), MaxValueValidator(1)], blank=True, null=True) - + country = models.ForeignKey(Country, on_delete=models.SET_NULL,related_name="institute_countrys", blank=True, null=True) quota = models.ForeignKey(Quota, on_delete=models.SET_NULL,related_name="institute_quotas", blank=True, null=True) #representative_user = models.ForeignKey("PyrosUser", on_delete=models.DO_NOTHING,related_name="institutes",default=1) @@ -475,9 +474,7 @@ class ScientificProgram(models.Model): science_theme = models.ForeignKey(ScienceTheme, on_delete=models.DO_NOTHING, related_name="scientific_program_theme", default=1) is_auto_validated = models.BooleanField(default=False) objects = ScientificProgramManager() - quota = models.ForeignKey(Quota, on_delete=models.SET_NULL, related_name="scientific_program_quotas", blank=True, null=True) - quota_f = models.FloatField( - validators=[MinValueValidator(0), MaxValueValidator(1)], blank=True, null=True) + class Meta: managed = True @@ -541,6 +538,9 @@ class SP_Period(models.Model): token = models.PositiveIntegerField(default=0) token_allocated = models.PositiveIntegerField(default=0, blank=True) token_remaining = models.PositiveIntegerField(default=0, blank=True) + quota = models.ForeignKey(Quota, on_delete=models.SET_NULL, related_name="sp_period_quotas", blank=True, null=True) + quota_f = models.FloatField( + validators=[MinValueValidator(0), MaxValueValidator(1)], blank=True, null=True) # Unit PI donne un nombre de priorité (100 = alert) priority = models.IntegerField( validators=[MinValueValidator(0), MaxValueValidator(100)], blank=True, null=True) -- libgit2 0.21.2