From 053162418ca3121e036bb758be73e3f80c973f32 Mon Sep 17 00:00:00 2001 From: Alexis Koralewski Date: Tue, 3 May 2022 17:18:42 +0200 Subject: [PATCH] Adding AgentSST, copy agentSP, agentScheduler and agentM to privatedev/plugin. Updating schema of observatory configuration, updating obs config of guitalens and tnc. Fixing error while trying to get 'is_real' attribute of an agent. Add bash script to copy content of privatedev into private folder. --- config/schemas/schema_observatory-2.0.yml | 2 ++ cp_private_dev_to_private.sh | 1 + privatedev/config/guitalens/observatory_guitalens.yml | 14 ++++++++++++++ privatedev/config/tnc/observatory_tnc.yml | 29 ++++++++++++++++++++--------- privatedev/plugin/agent/AgentSP.py | 238 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ privatedev/plugin/agent/AgentScheduler.py | 134 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ privatedev/plugin/agent_devices/AgentM.py | 275 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ pyros.py | 12 ++++++++++-- src/core/pyros_django/agent/Agent.py | 8 +++++++- src/core/pyros_django/agent/AgentSST.py | 122 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/core/pyros_django/dashboard/views.py | 1 - 11 files changed, 823 insertions(+), 13 deletions(-) create mode 100755 cp_private_dev_to_private.sh create mode 100755 privatedev/plugin/agent/AgentSP.py create mode 100755 privatedev/plugin/agent/AgentScheduler.py create mode 100755 privatedev/plugin/agent_devices/AgentM.py create mode 100644 src/core/pyros_django/agent/AgentSST.py diff --git a/config/schemas/schema_observatory-2.0.yml b/config/schemas/schema_observatory-2.0.yml index bb5629b..b10c99e 100644 --- a/config/schemas/schema_observatory-2.0.yml +++ b/config/schemas/schema_observatory-2.0.yml @@ -53,6 +53,8 @@ schema;schema_AGENT: required: True path: type: str + protocol: + type: str schema;schema_ALBUMS: type: seq diff --git a/cp_private_dev_to_private.sh b/cp_private_dev_to_private.sh new file mode 100755 index 0000000..3848631 --- /dev/null +++ b/cp_private_dev_to_private.sh @@ -0,0 +1 @@ +cp -r ./privatedev/* ./private/ \ No newline at end of file diff --git a/privatedev/config/guitalens/observatory_guitalens.yml b/privatedev/config/guitalens/observatory_guitalens.yml index d705fac..6c30d83 100644 --- a/privatedev/config/guitalens/observatory_guitalens.yml +++ b/privatedev/config/guitalens/observatory_guitalens.yml @@ -133,6 +133,20 @@ OBSERVATORY: name: majordome computer: MainComputer path: ~ + + + - AGENT: + name: AgentSP + computer: MainComputer + protocol: private/plugin/agent/AgentSP.py + + + - AGENT: + name: AgentScheduler + computer: MainComputer + protocol: private/plugin/agent/AgentScheduler.py + + TOPOLOGY: diff --git a/privatedev/config/tnc/observatory_tnc.yml b/privatedev/config/tnc/observatory_tnc.yml index 7307ebd..ba6d0ed 100644 --- a/privatedev/config/tnc/observatory_tnc.yml +++ b/privatedev/config/tnc/observatory_tnc.yml @@ -136,16 +136,16 @@ OBSERVATORY: computer: MainComputer path: private/plugin/agent_devices device: TAROT_meteo - protocol: private/plugin/agent_devices/plc_protocol.py + protocol: private/plugin/agent_devices/AgentM.py is_real: False - - AGENT_DEVICE: - name: AgentScheduler - computer: MainComputer - path: private/plugin/agent_devices - device: TAROT_meteo - protocol: private/plugin/agent_devices/plc_protocol.py - is_real: False + # - AGENT_DEVICE: + # name: AgentScheduler + # computer: MainComputer + # path: private/plugin/agent_devices + # device: TAROT_meteo + # protocol: private/plugin/agent_devices/plc_protocol.py + # is_real: False - AGENT_DEVICE: name: mount @@ -252,7 +252,18 @@ OBSERVATORY: name: majordome computer: MainComputer path: ~ - + + - AGENT: + name: AgentSP + computer: MainComputer + protocol: private/plugin/agent/AgentSP.py + + + - AGENT: + name: AgentScheduler + computer: MainComputer + protocol: private/plugin/agent/AgentScheduler.py + TOPOLOGY: diff --git a/privatedev/plugin/agent/AgentSP.py b/privatedev/plugin/agent/AgentSP.py new file mode 100755 index 0000000..6c6c89e --- /dev/null +++ b/privatedev/plugin/agent/AgentSP.py @@ -0,0 +1,238 @@ +import sys + +sys.path.append("..") +sys.path.append("../../../..") +from src.core.pyros_django.agent.Agent import Agent, build_agent +from common.models import Period, SP_Period, PyrosUser, SP_Period_Guest, SP_PeriodWorkflow, ScientificProgram,SP_Period_User, ScienceTheme +from django.shortcuts import reverse +from django.conf import settings +from django.core.mail import send_mail +from dateutil.relativedelta import relativedelta +from django.db.models import Q +from django.utils import timezone +from django.test.utils import setup_test_environment +import numpy as np + +class AgentSP(Agent): + + period = None + + # old config init + # def __init__(self, config_filename=None, RUN_IN_THREAD=True,use_db_test=False): + # ##if name is None: name = self.__class__.__name__ + # if use_db_test: + # print("USE DB TEST") + # setup_test_environment() + # self.TEST_COMMANDS_LIST = [""] + # super().__init__(None, RUN_IN_THREAD) + # next_period = Period.objects.next_period() + # period = next_period + + # new init with obsconfig + def __init__(self, RUN_IN_THREAD=True,use_db_test=False): + ##if name is None: name = self.__class__.__name__ + if use_db_test: + print("USE DB TEST") + setup_test_environment() + super().__init__(RUN_IN_THREAD) + next_period = Period.objects.next_period() + period = next_period + + # @override + def init(self): + super().init() + + def associate_tac_sp_auto(self,themes,tac_users,scientific_programs): + print("Associating tac to sp") + matrix_tac_themes = np.zeros([len(tac_users),len(themes)]) + maxtrix_themes_sp = np.zeros([len(themes),len(scientific_programs)]) + matrix_tac_sp = np.zeros([len(tac_users),len(scientific_programs)]) + for i,tac_user in enumerate(tac_users): + for j,theme in enumerate(themes): + if theme.name in tac_user.get_referee_themes_as_str(): + matrix_tac_themes[i,j] = 1 + for i,theme in enumerate(themes): + for j,sp in enumerate(scientific_programs): + if theme.id == sp.science_theme.id: + maxtrix_themes_sp[i,j] = 1 + matrix_tac_sp = np.dot(matrix_tac_themes,maxtrix_themes_sp) + nb_tac_per_sp = np.sum(matrix_tac_sp,axis=0) + next_period = Period.objects.next_period() + for i,sp in enumerate(scientific_programs): + if nb_tac_per_sp[i-1] == 2: + # We auto assign the tac users to scientific programs + print(sp) + sp_period = SP_Period.objects.get(scientific_program=sp,period=next_period) + available_tac_users = PyrosUser.objects.filter(referee_themes=sp.science_theme) + print("available tacs :") + print(available_tac_users) + sp_period.referee1 = available_tac_users[0] + sp_period.referee2 = available_tac_users[1] + sp_period.save() + #return matrix_tac_sp + + def change_sp_status(self,scientific_programs,new_status): + print(f"---- CHANGE STATUS FOR {scientific_programs} TO {new_status}------- ") + for sp in scientific_programs: + if sp.status != new_status: + sp.status = new_status + sp.save() + + def send_mail_to_tac_for_evaluation(self,tac_users,next_period): + domain = settings.DEFAULT_DOMAIN + url = f"{domain}{reverse('list_submitted_scientific_program')}" + mail_subject = '[PyROS CC] The evaluation period is now opened' + mail_message = (f"Hi,\n\nYou can now evaluate scientific programs for the next period ({next_period}).\n" + f"Click on the following link {url} to evaluate your assignated scientific programs." + "\n\nCordially,\n\nPyROS Control Center") + email_list = tac_users.values_list("email") + for email in email_list: + send_mail( + mail_subject, + mail_message, + from_email=None, + recipient_list=[email], + fail_silently=False, + ) + + def send_mail_to_observers_for_notification(self,sp_periods): + for sp_period in sp_periods: + sp_pi = sp_period.scientific_program.sp_pi + scientific_program = sp_period.scientific_program + domain = settings.DEFAULT_DOMAIN + url = f"{domain}{reverse('sp_register',args=(scientific_program.pk,sp_period.period.pk))}" + mail_subject = '[PyROS CC] New registration to a scientific program' + mail_message = (f"Hi,\n\nYou were invited to join a scientific program that as been submitted using PyROS.\n" + f"The name of the scientific program is {scientific_program.name} and his PI is {sp_pi.first_name} {sp_pi.last_name}.\n" + f"To accept this invitation, click on the following link : {url}\n" + f"Once you have joined the scientific program, you can start to submit sequences" + "You might be asked to login first and will be redirected to the scientific program page.\n" + "If the redirection doesn't work, click again on the link after you've logged in.\n" + "If you don't own an PyROS account, go on the website in order to create an account with the same mail adress that you are using to read this mail." + "\n\nCordially,\n\nPyROS Control Center") + invited_observers_of_sp = SP_Period_Guest.objects.filter(SP_Period=sp_period).values("user") + recipient_list = invited_observers_of_sp + for invited_observer in recipient_list: + send_mail( + mail_subject, + mail_message, + from_email=None, + recipient_list=[invited_observer], + fail_silently=False, + ) + + def send_mail_to_unit_users_for_tac_assignation(self): + domain = settings.DEFAULT_DOMAIN + url = f"{domain}{reverse('list_drafted_scientific_program')}" + mail_subject = '[PyROS CC] TAC assignation to scientific programs for the next period' + mail_message = (f"Hi,\n\nYou can assign TAC users to scientific programs by choosing them in the {url} page.\n" + "PyROS has suggested TAC to some of the scientific programs but you can change those assignations.\n" + f"The TAC assignation will be effective and couldn't be modified at the {self.period.submission_end_date}.\n" + "\n\nCordially,\n\nPyROS Control Center") + unit_users = PyrosUser.objects.unit_users().values_list("email",flat=True) + send_mail( + mail_subject, + mail_message, + from_email=None, + recipient_list=unit_users, + fail_silently=False, + ) + print("--------- SEND MAIL TO UNIT USERS ----------") + + + def automatic_period_workflow(self): + today = timezone.now().date() + next_period = Period.objects.next_period() + # check if next_period has changed + if self.period != next_period: + self.period = next_period + # get scientific program for next_period + next_sp = SP_Period.objects.filter(period=next_period) + auto_validated_sp = ScientificProgram.objects.filter(is_auto_validated=True) + auto_validated_sp_periods = SP_Period.objects.filter(scientific_program__in=auto_validated_sp,period=next_period) + # remove auto validated sp from next_sp + next_sp = next_sp.exclude(scientific_program__in=auto_validated_sp) + # get all tac users + tac_users = PyrosUser.objects.filter(user_level__name="TAC") + # submission workflow + if not SP_PeriodWorkflow.objects.filter(action=SP_PeriodWorkflow.SUBMISSION, period=self.period).exists(): + print("routine automatic period workflow SUBMISSION") + # if the next_period is actually in the "submission" subperiod + if next_period in Period.objects.submission_periods(): + # we have to assign TAC to SP + themes = ScienceTheme.objects.all() + # get id of scientific programs from SP_Period + sp_id = next_sp.exclude(scientific_program__is_auto_validated=True).filter(Q(referee1=None)|Q(referee2=None)).values("scientific_program") + # get scientific programs + sp = ScientificProgram.objects.filter(id__in=sp_id).order_by("name") + # if we are ten days before the end of the submission period, we have to assign TAC to scientific programs + # and send a mail to the Unit users to they assign the TAC users to SP + if next_period.submission_end_date + relativedelta(days=-10) == today : + self.associate_tac_sp_auto(themes,tac_users,sp) + # send mail to unit pi to tell him to associate TAC to SP + self.send_mail_to_unit_users_for_tac_assignation() + SP_PeriodWorkflow.objects.create(period=self.period,action=SP_PeriodWorkflow.SUBMISSION) + + if not SP_PeriodWorkflow.objects.filter(action=SP_PeriodWorkflow.EVALUATION, period=self.period).exists(): + print("routine automatic period workflow EVALUATION") + if next_period in Period.objects.evaluation_periods() and next_period.submission_end_date == today : + next_sp = SP_Period.objects.filter(period=next_period).exclude(scientific_program__in=auto_validated_sp).filter(status=SP_Period.STATUSES_DRAFT) + self.change_sp_status(next_sp,SP_Period.STATUSES_SUBMITTED) + self.send_mail_to_tac_for_evaluation(tac_users,next_period) + + # for auto validated sp, we have to change their status + self.change_sp_status(auto_validated_sp_periods,SP_Period.STATUSES_ACCEPTED) + for sp in auto_validated_sp_periods: + sp.is_valid = SP_Period.IS_VALID_ACCEPTED + sp.save() + + SP_PeriodWorkflow.objects.create(period=self.period,action=SP_PeriodWorkflow.EVALUATION) + if not SP_PeriodWorkflow.objects.filter(action=SP_PeriodWorkflow.VALIDATION, period=self.period).exists(): + print("routine automatic period workflow VALIDATION") + if next_period.unit_pi_validation_start_date == today : + next_sp = SP_Period.objects.filter(period=next_period).exclude(scientific_program__in=auto_validated_sp).filter(status=SP_Period.STATUSES_SUBMITTED) + self.change_sp_status(next_sp,SP_Period.STATUSES_EVALUATED) + next_sp = SP_Period.objects.filter(period=next_period).exclude(scientific_program__in=auto_validated_sp).filter(status=SP_Period.STATUSES_EVALUATED) + SP_PeriodWorkflow.objects.create(period=self.period,action=SP_PeriodWorkflow.VALIDATION) + if not SP_PeriodWorkflow.objects.filter(action=SP_PeriodWorkflow.NOTIFICATION, period=self.period).exists(): + print("routine automatic period workflow NOTIFICATION") + if next_period in Period.objects.notification_periods(): + next_sp_accepted = SP_Period.objects.filter(period=next_period).filter(is_valid=SP_Period.IS_VALID_ACCEPTED) + self.change_sp_status(next_sp_accepted,SP_Period.STATUSES_ACCEPTED) + next_sp_rejected = SP_Period.objects.filter(period=next_period).filter(is_valid=SP_Period.IS_VALID_REJECTED) + self.change_sp_status(next_sp_rejected,SP_Period.STATUSES_REJECTED) + next_sp_to_be_notified = next_sp.filter(status=SP_Period.STATUSES_ACCEPTED,is_valid = True) + self.send_mail_to_observers_for_notification(next_sp_to_be_notified) + SP_PeriodWorkflow.objects.create(period=self.period,action=SP_PeriodWorkflow.NOTIFICATION) + + def routine_process_body(self): + print("routine automatic period workflow") + print(SP_PeriodWorkflow.objects.all()) + print(PyrosUser.objects.all()) + for sp_period_workflow in SP_PeriodWorkflow.objects.all(): + print(sp_period_workflow.period) + print(sp_period_workflow.action) + self.automatic_period_workflow() + + +if __name__ == "__main__": + + # with thread + RUN_IN_THREAD=True + # with process + #RUN_IN_THREAD=False + print("ARGV OF AGENT SP :",sys.argv) + if len(sys.argv) > 1 and sys.argv[1] == "test": + print("i'm in test") + agentSP = AgentSP(use_db_test=True) + agentSP.run() + #agent = build_agent(agentSP, RUN_IN_THREAD=True) + else: + agent = build_agent(AgentSP, RUN_IN_THREAD=RUN_IN_THREAD) + ''' + TEST_MODE, configfile = extract_parameters() + agent = AgentM("AgentM", configfile, RUN_IN_THREAD) + agent.setSimulatorMode(TEST_MODE) + ''' + print(agent) + agent.run() diff --git a/privatedev/plugin/agent/AgentScheduler.py b/privatedev/plugin/agent/AgentScheduler.py new file mode 100755 index 0000000..b907f07 --- /dev/null +++ b/privatedev/plugin/agent/AgentScheduler.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 + +import sys +##import utils.Logger as L +#import threading #, multiprocessing, os +import time + +#from django.db import transaction +#from common.models import Command + +sys.path.append("..") +sys.path.append("../../../..") +from src.core.pyros_django.agent.Agent import Agent, build_agent, log + +# PM 20190416 recycle code +#from common.models import * +from common.models import Sequence + +##log = L.setupLogger("AgentXTaskLogger", "AgentX") + + + +class AgentScheduler(Agent): + + + # FOR TEST ONLY + # Run this agent in simulator mode + TEST_MODE = False + # Run the assertion tests at the end + TEST_WITH_FINAL_TEST = True + TEST_MAX_DURATION_SEC = None + #TEST_MAX_DURATION_SEC = 120 + + # PM 20190416 fucking config path starting: /home/patrick/Dev/PYROS/start_agent.py agentM + ##_path_data = 'config' + _path_data = 'config/old_config' + + log.debug("PLC instanciated") + + + + ''' + # Who should I send commands to ? + #TEST_COMMANDS_DEST = "myself" + TEST_COMMANDS_DEST = "AgentA" + # Scenario to be executed + TEST_COMMANDS_LIST = [ + "go_active", + "go_idle", + "go_active", + "go_idle", + "go_active", + "go_idle", + "exit", + ] + ''' + + """ + ================================================================= + FUNCTIONS RUN INSIDE MAIN THREAD + ================================================================= + """ + # old config + # @override + #def __init__(self, name:str=None, config_filename=None, RUN_IN_THREAD=True): + # def __init__(self, config_filename=None, RUN_IN_THREAD=True): + # ##if name is None: name = self.__class__.__name__ + # super().__init__(config_filename, RUN_IN_THREAD) + + # new config (obsconfig) + def __init__(self, name:str=None, RUN_IN_THREAD=True): + if name is None: + name = self.__class__.__name__ + super().__init__(RUN_IN_THREAD) + # @override + def init(self): + super().init() + log.debug("end init()") + # --- Set the mode according the startmode value + ##agent_alias = self.__class__.__name__ + ##self.set_mode_from_config(agent_alias) + + ''' + # @override + def load_config(self): + super().load_config() + ''' + + ''' + # @override + def update_survey(self): + super().update_survey() + ''' + + ''' + # @override + def get_next_command(self): + return super().get_next_command() + ''' + + # @override + def do_log(self): + super().do_log() + + def replan_sequences(self): + print("\n start of sequences (re-)planning...\n") + time.sleep(5) + sequences = Sequence.objects.filter(status="TBP") + print("List of sequences to be planned :") + for seq in sequences: + print('-', seq.name, '('+seq.status+')') + #print('-- with albums : ', seq.albums) + print("\n ...end of sequences (re-)planning\n") + + # Note : called by _routine_process() in Agent + # @override + def routine_process_body(self): + print("The Observatory configuration :") + self.show_config() + log.debug("in routine_process_body()") + self.replan_sequences() + + ''' + # @override + def exec_specific_cmd_end(self, cmd:Command, from_thread=True): + super().exec_specific_cmd_end(cmd, from_thread) + ''' + + +if __name__ == "__main__": + + agent = build_agent(AgentScheduler) + print(agent) + agent.run() diff --git a/privatedev/plugin/agent_devices/AgentM.py b/privatedev/plugin/agent_devices/AgentM.py new file mode 100755 index 0000000..fe5b654 --- /dev/null +++ b/privatedev/plugin/agent_devices/AgentM.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 + +import sys +##import utils.Logger as L +#import threading #, multiprocessing, os +#import time + +#from django.db import transaction +#from common.models import Command +sys.path.append("..") +sys.path.append("../../../..") +from src.core.pyros_django.agent.Agent import Agent, build_agent, log + +# PM 20190416 recycle code +from src.core.pyros_django.devices.PLC import PLCController +from src.core.pyros_django.monitoring.plc_checker import PlcChecker +from common.models import * +from config.pyros.config_pyros import ConfigPyros + + + +##log = L.setupLogger("AgentXTaskLogger", "AgentX") + + + +class AgentM(Agent): + + + # FOR TEST ONLY + # Run this agent in simulator mode + TEST_MODE = False + # Run the assertion tests at the end + TEST_WITH_FINAL_TEST = True + TEST_MAX_DURATION_SEC = None + #TEST_MAX_DURATION_SEC = 120 + + # PM 20190416 fucking config path starting: /home/patrick/Dev/PYROS/start_agent.py agentM + ##_path_data = 'config' + _path_data = 'config/old_config' + # PM 20190416 recycle code + plcController = PLCController() + print ("AGENT ENV: config PLC is (ip={}, port={})".format(plcController.ip, plcController.port)) + plc_checker = PlcChecker() + + log.debug("PLC instanciated") + time_history_minutes = 4 + + + ''' + # Who should I send commands to ? + #TEST_COMMANDS_DEST = "myself" + TEST_COMMANDS_DEST = "AgentA" + # Scenario to be executed + TEST_COMMANDS_LIST = [ + "go_active", + "go_idle", + "go_active", + "go_idle", + "go_active", + "go_idle", + "exit", + ] + ''' + + """ + ================================================================= + FUNCTIONS RUN INSIDE MAIN THREAD + ================================================================= + """ + # old config + # @override + #def __init__(self, name:str=None, config_filename=None, RUN_IN_THREAD=True): + # def __init__(self, config_filename=None, RUN_IN_THREAD=True): + # ##if name is None: name = self.__class__.__name__ + # super().__init__(config_filename, RUN_IN_THREAD) + + # new config (obsconfig) + def __init__(self, name:str=None, RUN_IN_THREAD=True): + if name is None: + name = self.__class__.__name__ + super().__init__(RUN_IN_THREAD) + PYROS_CONFIG_FILE = os.environ.get("pyros_config_file") + if PYROS_CONFIG_FILE: + CONFIG_PYROS = ConfigPyros(PYROS_CONFIG_FILE).pyros_config + self.time_history_minutes = int(CONFIG_PYROS.get("ENV").get("time_history")) + log.info(f"time_history_minutes set to {int(self.time_history_minutes)}") + # @override + def init(self): + super().init() + + log.debug("end init()") + # --- Set the mode according the startmode value + ##agent_alias = self.__class__.__name__ + ##self.set_mode_from_config(agent_alias) + + ''' + # @override + def load_config(self): + super().load_config() + ''' + + ''' + # @override + def update_survey(self): + super().update_survey() + ''' + + ''' + # @override + def get_next_command(self): + return super().get_next_command() + ''' + + # @override + def do_log(self): + super().do_log() + + + + """ + ================================================================= + FUNCTIONS RUN INSIDE A SUB-THREAD (OR A PROCESS) (thread_*()) + ================================================================= + """ + + # Define your own command step(s) here + def cmd_step1(self, step:int): + cmd = self._current_specific_cmd + cmd.result = f"in step #{step}/{self._thread_total_steps_number}" + cmd.save() + """ + if self.RUN_IN_THREAD: + print("(save from thread)") + cmd.save() + else: + #@transaction.atomic + print("(save from process)") + with transaction.atomic(): + cmd.save() + #Command.objects.select_for_update() + """ + + def cmd_step2(self, step:int): + self.cmd_step1(step) + def cmd_step3(self, step:int): + self.cmd_step1(step) + def cmd_step4(self, step:int): + self.cmd_step1(step) + + """ + # @override + def thread_exec_specific_cmd_step(self, step:int, sleep_time:float=1.0): + self.thread_stop_if_asked() + cmd = self._current_specific_cmd + print(f">>>>> Thread (cmd {cmd.name}): step #{step}/5") + self.sleep(sleep_time) + """ + + ''' + # @override + def exec_specific_cmd_start(self, cmd:Command, from_thread=True): + super().exec_specific_cmd_start(cmd, from_thread) + ''' + + # @override + # previous name of function : routine_process + # Note : in Agent.py, routine_process_body seems to be the main function of routine of the agent + # We need to override routine_process_body and not routine_process + def routine_process_body(self): + log.debug("in routine_process_body()") + print("TODO: we recycle code") + status_plc = self.plcController.getStatus() + if self.parseNewStatus(status_plc): + self.saveWeather() + #self.saveInternalMonitoring() + + def parseNewStatus(self,status_plc): + # """ PM 20181009 parse new status for config + # Find return string "plc_status" positin within status_plc + if status_plc.find('PLC_STATUS') >= 0: + self.plc_checker.chk_config(status_plc) + return True + return False + + def saveWeather(self): + outside = WeatherWatch() + inside = SiteWatch() + datetimenow = datetime.now(timezone.utc) + latest_entry_of_history = WeatherWatchHistory.objects.all().order_by("-datetime").first() + if latest_entry_of_history != None: + # Get last entry of WeatherWatchHistory as WeatherWatch + latest_entry_of_history_as_weather = WeatherWatch.objects.get(id=latest_entry_of_history.weather.id) + outside_attributes_values = {} + for sensor in self.plc_checker.monitoring_names.keys(): + if sensor in self.plc_checker.inside_sensors: + value = self.plc_checker.get_sensor(sensor) + inside.setAttribute(sensor,value) + else: + value = self.plc_checker.get_sensor(sensor) + outside.setAttribute(sensor, value) + outside_attributes_values[sensor] = value + outside.setGlobalStatus() + outside.save() + #inside.save() + # We don't have an history for weatherwatch + if latest_entry_of_history == None: + weather_history = WeatherWatchHistory() + weather_history.weather = outside + for sensor in outside_attributes_values.keys(): + weather_history.setAttribute(sensor,outside_attributes_values.get(sensor)) + # save also sensors + weather_history.save() + else: + time_between_history_and_latest_entry = datetimenow - latest_entry_of_history_as_weather.updated + sec_diff = time_between_history_and_latest_entry.total_seconds() / 60 + # if diff between last entry of history and current time if greather than x then we save a new entry in history + if int(sec_diff) > self.time_history_minutes: + weather_history = WeatherWatchHistory() + weather_history.weather = outside + for sensor in outside_attributes_values.keys(): + weather_history.setAttribute(sensor,outside_attributes_values.get(sensor)) + weather_history.save() + log.debug("saved weather") + + def isInsideMonitoring(self,key): + print(key) + if key in ("Power_input","Roof_state"): + return True + else: + return False + # @override + def thread_exec_specific_cmd_main(self): + # This is optional + self.thread_set_total_steps_number(5) + + # HERE, write your own scenario + + # scenario OK + self.thread_exec_specific_cmd_step(1, self.cmd_step1, 1) + self.thread_exec_specific_cmd_step(2, self.cmd_step2, 3) + self.thread_exec_specific_cmd_step(3, self.cmd_step1, 5) + self.thread_exec_specific_cmd_step(4, self.cmd_step3, 10) + self.thread_exec_specific_cmd_step(5, self.cmd_step1, 4) + # ... as many as you need + + """ autre scenario + self.thread_exec_specific_cmd_step(1, self.cmd_step1, 1) + self.thread_exec_specific_cmd_step(2, self.cmd_step2, 2) + self.thread_exec_specific_cmd_step(3, self.cmd_step1, 2) + self.thread_exec_specific_cmd_step(4, self.cmd_step3, 2) + self.thread_exec_specific_cmd_step(5, self.cmd_step1, 3) + """ + + ''' + # @override + def exec_specific_cmd_end(self, cmd:Command, from_thread=True): + super().exec_specific_cmd_end(cmd, from_thread) + ''' + + +if __name__ == "__main__": + + # with thread + RUN_IN_THREAD=True + # with process + #RUN_IN_THREAD=False + + agent = build_agent(AgentM, RUN_IN_THREAD=RUN_IN_THREAD) + ''' + TEST_MODE, configfile = extract_parameters() + agent = AgentM("AgentM", configfile, RUN_IN_THREAD) + agent.setSimulatorMode(TEST_MODE) + ''' + print(agent) + agent.run() diff --git a/pyros.py b/pyros.py index 97fc929..62ea4ce 100755 --- a/pyros.py +++ b/pyros.py @@ -63,6 +63,7 @@ AGENTS = { "agentA": "AgentA", "agentB": "AgentB", "agentC": "AgentC", + "agentSST": "agent", # "agentDevice" : "AgentDevice", # "agentDeviceTelescopeGemini" : "AgentDeviceTelescopeGemini", "agentDeviceGemini": "AgentDeviceGemini", @@ -96,6 +97,9 @@ if IS_WINDOWS: PYTHON = "python.exe" +PROJECT_ROOT_PATH = os.getcwd() +os.environ["PROJECT_ROOT_PATH"] = PROJECT_ROOT_PATH + try: # WITH_DOCKER is an environment varialbe from our Docker image WITH_DOCKER = os.environ['WITH_DOCKER'] @@ -783,10 +787,11 @@ def initdb(): @click.option('--configfile', '-c', help='the configuration file to be used') @click.option('--observatory', '-o', help='the observatory name to be used') @click.option('--unit', '-u', help='the unit name to be used') +@click.option("--computer_hostname","-cp", help="The name of simulated computer hostname") # @click.option('--format', '-f', type=click.Choice(['html', 'xml', 'text']), default='html', show_default=True) # @click.option('--port', default=8000) # def start(agent:str, configfile:str, test, verbosity): -def start(agent: str, configfile: str, observatory: str, unit: str): +def start(agent: str, configfile: str, observatory: str, unit: str, computer_hostname: str): log.debug("Running start command") try: from config.pyros.config_pyros import ConfigPyros @@ -923,7 +928,8 @@ def start(agent: str, configfile: str, observatory: str, unit: str): ''' if agent_name in ["agentM", "agentSP", "agentScheduler"]: os.chdir(PYROS_DJANGO_BASE_DIR + os.sep + agent_folder) - + else: + os.chdir(PYROS_DJANGO_BASE_DIR+"/agent/") # cmd = "-m AgentX" # cmd = f" Agent{agent_name[5:]}.py {configfile}" cmd = f"Agent{agent_name[5:]}.py" @@ -937,6 +943,8 @@ def start(agent: str, configfile: str, observatory: str, unit: str): cmd += " -v" if configfile: cmd += " {configfile}" + if computer_hostname: + cmd += " -c {computer_hostname}" # if not test_mode(): current_processes.append( [execProcessFromVenvAsync(cmd), agent_name, -1] ) # Append this process ( [process id, agent_name, result=failure] ) diff --git a/src/core/pyros_django/agent/Agent.py b/src/core/pyros_django/agent/Agent.py index 71be8ff..52c28b8 100755 --- a/src/core/pyros_django/agent/Agent.py +++ b/src/core/pyros_django/agent/Agent.py @@ -1854,7 +1854,13 @@ def build_agent(Agent_type:Agent, RUN_IN_THREAD=True): return agent #agent = Agent_type(name, configfile, RUN_IN_THREAD) # Get the information of the agent name (name of class) within obsconfig and get the "is_real" attribute - WITH_SIM = not agent.get_config().get_agent_information(agent.unit,agent.name)["is_real"] + if agent.name in agent.get_config().get_agents(agent.unit).keys(): + if agent.get_config().get_agent_information(agent.unit,agent.name).get("is_real"): + WITH_SIM = not agent.get_config().get_agent_information(agent.unit,agent.name)["is_real"] + else: + WITH_SIM = True + else: + WITH_SIM = True agent._set_with_simulator(WITH_SIM) agent._set_test_mode(TEST_MODE) #print(logger) diff --git a/src/core/pyros_django/agent/AgentSST.py b/src/core/pyros_django/agent/AgentSST.py new file mode 100644 index 0000000..892f581 --- /dev/null +++ b/src/core/pyros_django/agent/AgentSST.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import subprocess +import sys, os +##import utils.Logger as L +#import threading #, multiprocessing, os +import time + +#from django.db import transaction +#from common.models import Command + +from Agent import Agent, build_agent, log +import socket + + +class AgentSST(Agent): + computer = "MainComputer" + _previous_dir = "" + PROJECT_ROOT_PATH = "" + VENV_PYTHON = "" + subprocess_dict = {} + + + def __init__(self, name:str=None, RUN_IN_THREAD=True,sim_computer=None): + + super().__init__(RUN_IN_THREAD) + self.PROJECT_ROOT_PATH = os.environ["PROJECT_ROOT_PATH"] + if name is None: + name = self.__class__.__name__ + # if sim_computer: + # self.computer = sim_computer + # else: + # self.computer = socket.gethostname() + WITH_DOCKER = False + if os.environ.get("WITH_DOCKER"): + WITH_DOCKER = True + if WITH_DOCKER: + VENV_ROOT = "" + VENV = "" + VENV_BIN = "" + else: + VENV_ROOT = "venv" + VENV = "venv_py3_pyros" + VENV_BIN = ( + self.PROJECT_ROOT_PATH + + os.sep + VENV_ROOT + + os.sep + VENV + + os.sep + "bin" + + os.sep + ) + VENV_PYTHON = VENV_BIN + "python3" + log.info(f"PC hostname is {self.computer}") + self.start_agents() + + def start_agents(self,agent_name=None): + """ + Start all agents or one agent of obs_config + + Args: + agent_name (_type_, optional): Specific agent name to start. Defaults to None. + """ + obs_config = self.get_config() + + agents = obs_config.get_agents_per_computer(obs_config.unit_name).get(self.computer) + if agents is None: + print("Computer not found in obs config") + exit(1) + #self.change_dir(self.PROJECT_ROOT_PATH) + if agent_name: + agent = agent_name + # Start a specific agent of obs_config (restart) + agent_informations = obs_config.get_agent_information(obs_config.unit_name,agent) + protocol = agent_informations.get("protocol") + if protocol: + protocol_folder_abs_path = os.path.join(self.PROJECT_ROOT_PATH, os.path.dirname(protocol)) + + protocol_script_name = protocol.split("/")[-1] + if os.path.exists(protocol_folder_abs_path + os.sep + protocol_script_name): + cmd = self.VENV_PYTHON + protocol_folder_abs_path + os.sep + protocol_script_name + + process = subprocess.Popen(f"{cmd}", shell=True, stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT) + self.subprocess_dict[agent] = process + log.info(f"Start agent {agent} with cmd {cmd}") + + else: + # Start every agent of obs_config (initial start) + for agent in agents: + agent_informations = obs_config.get_agent_information(obs_config.unit_name,agent) + protocol = agent_informations.get("protocol") + if protocol: + protocol_folder_abs_path = os.path.join(self.PROJECT_ROOT_PATH, os.path.dirname(protocol)) + + protocol_script_name = protocol.split("/")[-1] + if os.path.exists(protocol_folder_abs_path + os.sep + protocol_script_name): + cmd = self.VENV_PYTHON + protocol_folder_abs_path + os.sep + protocol_script_name + + process = subprocess.Popen(f"{cmd}", shell=True, stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT) + self.subprocess_dict[agent] = process + log.info(f"Start agent {agent} with cmd {cmd}") + + def start_agent(self, agent_name:str): + """ + Start a specific agent of obs_config (Restart) + + Args: + agent_name (str): Name of agent to start + """ + self.start_agents(agent_name) + + def routine_process_body(self): + for process in self.subprocess_dict.values(): + process.terminate() + process.wait() + exit(0) + # needs to receive new commands to stop or start new agents + #start_agents(self) + +if __name__ == "__main__": + + agent = build_agent(AgentSST) + agent.run() \ No newline at end of file diff --git a/src/core/pyros_django/dashboard/views.py b/src/core/pyros_django/dashboard/views.py index 10a2706..272e990 100644 --- a/src/core/pyros_django/dashboard/views.py +++ b/src/core/pyros_django/dashboard/views.py @@ -211,7 +211,6 @@ def get_lastest_and_last_x_minutes_before_latest_weather(x:float=0): # If we have entries in WeatherWatchHistory if WeatherWatchHistory.objects.all().exists(): # Get the first entry in history that is less or equal than time_start_range - # TODO : améliorer la récupération du first_weather_of_time_start -> peut ne pas respecter le temps - x donné first_weather_of_time_start = WeatherWatch.objects.filter(updated__lte=time_start_range).order_by("-updated").first() if first_weather_of_time_start != None: first_weather_of_time_start_id = first_weather_of_time_start.id -- libgit2 0.21.2