PYROS-IRAP / pyros

Commit 053162418ca3121e036bb758be73e3f80c973f32

Authored by Alexis Koralewski
1 parent bbf0ac6e
Exists in dev

Adding AgentSST, copy agentSP, agentScheduler and agentM to privatedev/plugin. U… 

…pdating schema of observatory configuration, updating obs config of guitalens and tnc. Fixing error while trying to get 'is_real' attribute of an agent. Add bash script to copy content of privatedev into private folder.
Showing 11 changed files with 823 additions and 13 deletions   Show diff stats
Inline Side-by-side
config/schemas/schema_observatory-2.0.yml
  Diff comments   View file @0531624
... ... @@ -53,6 +53,8 @@ schema;schema_AGENT:
53 53 required: True
54 54 path:
55 55 type: str
  56 + protocol:
  57 + type: str
56 58  
57 59 schema;schema_ALBUMS:
58 60 type: seq
... ...
cp_private_dev_to_private.sh 0 → 100755
  Diff comments   View file @0531624
... ... @@ -0,0 +1 @@
  1 +cp -r ./privatedev/* ./private/
0 2 \ No newline at end of file
... ...
privatedev/config/guitalens/observatory_guitalens.yml
  Diff comments   View file @0531624
... ... @@ -133,6 +133,20 @@ OBSERVATORY:
133 133 name: majordome
134 134 computer: MainComputer
135 135 path: ~
  136 +
  137 +
  138 + - AGENT:
  139 + name: AgentSP
  140 + computer: MainComputer
  141 + protocol: private/plugin/agent/AgentSP.py
  142 +
  143 +
  144 + - AGENT:
  145 + name: AgentScheduler
  146 + computer: MainComputer
  147 + protocol: private/plugin/agent/AgentScheduler.py
  148 +
  149 +
136 150  
137 151 TOPOLOGY:
138 152  
... ...
privatedev/config/tnc/observatory_tnc.yml
  Diff comments   View file @0531624
... ... @@ -136,16 +136,16 @@ OBSERVATORY:
136 136 computer: MainComputer
137 137 path: private/plugin/agent_devices
138 138 device: TAROT_meteo
139   - protocol: private/plugin/agent_devices/plc_protocol.py
  139 + protocol: private/plugin/agent_devices/AgentM.py
140 140 is_real: False
141 141  
142   - - AGENT_DEVICE:
143   - name: AgentScheduler
144   - computer: MainComputer
145   - path: private/plugin/agent_devices
146   - device: TAROT_meteo
147   - protocol: private/plugin/agent_devices/plc_protocol.py
148   - is_real: False
  142 + # - AGENT_DEVICE:
  143 + # name: AgentScheduler
  144 + # computer: MainComputer
  145 + # path: private/plugin/agent_devices
  146 + # device: TAROT_meteo
  147 + # protocol: private/plugin/agent_devices/plc_protocol.py
  148 + # is_real: False
149 149  
150 150 - AGENT_DEVICE:
151 151 name: mount
... ... @@ -252,7 +252,18 @@ OBSERVATORY:
252 252 name: majordome
253 253 computer: MainComputer
254 254 path: ~
255   -
  255 +
  256 + - AGENT:
  257 + name: AgentSP
  258 + computer: MainComputer
  259 + protocol: private/plugin/agent/AgentSP.py
  260 +
  261 +
  262 + - AGENT:
  263 + name: AgentScheduler
  264 + computer: MainComputer
  265 + protocol: private/plugin/agent/AgentScheduler.py
  266 +
256 267  
257 268 TOPOLOGY:
258 269  
... ...
privatedev/plugin/agent/AgentSP.py 0 → 100755
  Diff comments   View file @0531624
... ... @@ -0,0 +1,238 @@
  1 +import sys
  2 +
  3 +sys.path.append("..")
  4 +sys.path.append("../../../..")
  5 +from src.core.pyros_django.agent.Agent import Agent, build_agent
  6 +from common.models import Period, SP_Period, PyrosUser, SP_Period_Guest, SP_PeriodWorkflow, ScientificProgram,SP_Period_User, ScienceTheme
  7 +from django.shortcuts import reverse
  8 +from django.conf import settings
  9 +from django.core.mail import send_mail
  10 +from dateutil.relativedelta import relativedelta
  11 +from django.db.models import Q
  12 +from django.utils import timezone
  13 +from django.test.utils import setup_test_environment
  14 +import numpy as np
  15 +
  16 +class AgentSP(Agent):
  17 +
  18 + period = None
  19 +
  20 + # old config init
  21 + # def __init__(self, config_filename=None, RUN_IN_THREAD=True,use_db_test=False):
  22 + # ##if name is None: name = self.__class__.__name__
  23 + # if use_db_test:
  24 + # print("USE DB TEST")
  25 + # setup_test_environment()
  26 + # self.TEST_COMMANDS_LIST = [""]
  27 + # super().__init__(None, RUN_IN_THREAD)
  28 + # next_period = Period.objects.next_period()
  29 + # period = next_period
  30 +
  31 + # new init with obsconfig
  32 + def __init__(self, RUN_IN_THREAD=True,use_db_test=False):
  33 + ##if name is None: name = self.__class__.__name__
  34 + if use_db_test:
  35 + print("USE DB TEST")
  36 + setup_test_environment()
  37 + super().__init__(RUN_IN_THREAD)
  38 + next_period = Period.objects.next_period()
  39 + period = next_period
  40 +
  41 + # @override
  42 + def init(self):
  43 + super().init()
  44 +
  45 + def associate_tac_sp_auto(self,themes,tac_users,scientific_programs):
  46 + print("Associating tac to sp")
  47 + matrix_tac_themes = np.zeros([len(tac_users),len(themes)])
  48 + maxtrix_themes_sp = np.zeros([len(themes),len(scientific_programs)])
  49 + matrix_tac_sp = np.zeros([len(tac_users),len(scientific_programs)])
  50 + for i,tac_user in enumerate(tac_users):
  51 + for j,theme in enumerate(themes):
  52 + if theme.name in tac_user.get_referee_themes_as_str():
  53 + matrix_tac_themes[i,j] = 1
  54 + for i,theme in enumerate(themes):
  55 + for j,sp in enumerate(scientific_programs):
  56 + if theme.id == sp.science_theme.id:
  57 + maxtrix_themes_sp[i,j] = 1
  58 + matrix_tac_sp = np.dot(matrix_tac_themes,maxtrix_themes_sp)
  59 + nb_tac_per_sp = np.sum(matrix_tac_sp,axis=0)
  60 + next_period = Period.objects.next_period()
  61 + for i,sp in enumerate(scientific_programs):
  62 + if nb_tac_per_sp[i-1] == 2:
  63 + # We auto assign the tac users to scientific programs
  64 + print(sp)
  65 + sp_period = SP_Period.objects.get(scientific_program=sp,period=next_period)
  66 + available_tac_users = PyrosUser.objects.filter(referee_themes=sp.science_theme)
  67 + print("available tacs :")
  68 + print(available_tac_users)
  69 + sp_period.referee1 = available_tac_users[0]
  70 + sp_period.referee2 = available_tac_users[1]
  71 + sp_period.save()
  72 + #return matrix_tac_sp
  73 +
  74 + def change_sp_status(self,scientific_programs,new_status):
  75 + print(f"---- CHANGE STATUS FOR {scientific_programs} TO {new_status}------- ")
  76 + for sp in scientific_programs:
  77 + if sp.status != new_status:
  78 + sp.status = new_status
  79 + sp.save()
  80 +
  81 + def send_mail_to_tac_for_evaluation(self,tac_users,next_period):
  82 + domain = settings.DEFAULT_DOMAIN
  83 + url = f"{domain}{reverse('list_submitted_scientific_program')}"
  84 + mail_subject = '[PyROS CC] The evaluation period is now opened'
  85 + mail_message = (f"Hi,\n\nYou can now evaluate scientific programs for the next period ({next_period}).\n"
  86 + f"Click on the following link {url} to evaluate your assignated scientific programs."
  87 + "\n\nCordially,\n\nPyROS Control Center")
  88 + email_list = tac_users.values_list("email")
  89 + for email in email_list:
  90 + send_mail(
  91 + mail_subject,
  92 + mail_message,
  93 + from_email=None,
  94 + recipient_list=[email],
  95 + fail_silently=False,
  96 + )
  97 +
  98 + def send_mail_to_observers_for_notification(self,sp_periods):
  99 + for sp_period in sp_periods:
  100 + sp_pi = sp_period.scientific_program.sp_pi
  101 + scientific_program = sp_period.scientific_program
  102 + domain = settings.DEFAULT_DOMAIN
  103 + url = f"{domain}{reverse('sp_register',args=(scientific_program.pk,sp_period.period.pk))}"
  104 + mail_subject = '[PyROS CC] New registration to a scientific program'
  105 + mail_message = (f"Hi,\n\nYou were invited to join a scientific program that as been submitted using PyROS.\n"
  106 + f"The name of the scientific program is {scientific_program.name} and his PI is {sp_pi.first_name} {sp_pi.last_name}.\n"
  107 + f"To accept this invitation, click on the following link : {url}\n"
  108 + f"Once you have joined the scientific program, you can start to submit sequences"
  109 + "You might be asked to login first and will be redirected to the scientific program page.\n"
  110 + "If the redirection doesn't work, click again on the link after you've logged in.\n"
  111 + "If you don't own an PyROS account, go on the website in order to create an account with the same mail adress that you are using to read this mail."
  112 + "\n\nCordially,\n\nPyROS Control Center")
  113 + invited_observers_of_sp = SP_Period_Guest.objects.filter(SP_Period=sp_period).values("user")
  114 + recipient_list = invited_observers_of_sp
  115 + for invited_observer in recipient_list:
  116 + send_mail(
  117 + mail_subject,
  118 + mail_message,
  119 + from_email=None,
  120 + recipient_list=[invited_observer],
  121 + fail_silently=False,
  122 + )
  123 +
  124 + def send_mail_to_unit_users_for_tac_assignation(self):
  125 + domain = settings.DEFAULT_DOMAIN
  126 + url = f"{domain}{reverse('list_drafted_scientific_program')}"
  127 + mail_subject = '[PyROS CC] TAC assignation to scientific programs for the next period'
  128 + mail_message = (f"Hi,\n\nYou can assign TAC users to scientific programs by choosing them in the {url} page.\n"
  129 + "PyROS has suggested TAC to some of the scientific programs but you can change those assignations.\n"
  130 + f"The TAC assignation will be effective and couldn't be modified at the {self.period.submission_end_date}.\n"
  131 + "\n\nCordially,\n\nPyROS Control Center")
  132 + unit_users = PyrosUser.objects.unit_users().values_list("email",flat=True)
  133 + send_mail(
  134 + mail_subject,
  135 + mail_message,
  136 + from_email=None,
  137 + recipient_list=unit_users,
  138 + fail_silently=False,
  139 + )
  140 + print("--------- SEND MAIL TO UNIT USERS ----------")
  141 +
  142 +
  143 + def automatic_period_workflow(self):
  144 + today = timezone.now().date()
  145 + next_period = Period.objects.next_period()
  146 + # check if next_period has changed
  147 + if self.period != next_period:
  148 + self.period = next_period
  149 + # get scientific program for next_period
  150 + next_sp = SP_Period.objects.filter(period=next_period)
  151 + auto_validated_sp = ScientificProgram.objects.filter(is_auto_validated=True)
  152 + auto_validated_sp_periods = SP_Period.objects.filter(scientific_program__in=auto_validated_sp,period=next_period)
  153 + # remove auto validated sp from next_sp
  154 + next_sp = next_sp.exclude(scientific_program__in=auto_validated_sp)
  155 + # get all tac users
  156 + tac_users = PyrosUser.objects.filter(user_level__name="TAC")
  157 + # submission workflow
  158 + if not SP_PeriodWorkflow.objects.filter(action=SP_PeriodWorkflow.SUBMISSION, period=self.period).exists():
  159 + print("routine automatic period workflow SUBMISSION")
  160 + # if the next_period is actually in the "submission" subperiod
  161 + if next_period in Period.objects.submission_periods():
  162 + # we have to assign TAC to SP
  163 + themes = ScienceTheme.objects.all()
  164 + # get id of scientific programs from SP_Period
  165 + sp_id = next_sp.exclude(scientific_program__is_auto_validated=True).filter(Q(referee1=None)|Q(referee2=None)).values("scientific_program")
  166 + # get scientific programs
  167 + sp = ScientificProgram.objects.filter(id__in=sp_id).order_by("name")
  168 + # if we are ten days before the end of the submission period, we have to assign TAC to scientific programs
  169 + # and send a mail to the Unit users to they assign the TAC users to SP
  170 + if next_period.submission_end_date + relativedelta(days=-10) == today :
  171 + self.associate_tac_sp_auto(themes,tac_users,sp)
  172 + # send mail to unit pi to tell him to associate TAC to SP
  173 + self.send_mail_to_unit_users_for_tac_assignation()
  174 + SP_PeriodWorkflow.objects.create(period=self.period,action=SP_PeriodWorkflow.SUBMISSION)
  175 +
  176 + if not SP_PeriodWorkflow.objects.filter(action=SP_PeriodWorkflow.EVALUATION, period=self.period).exists():
  177 + print("routine automatic period workflow EVALUATION")
  178 + if next_period in Period.objects.evaluation_periods() and next_period.submission_end_date == today :
  179 + next_sp = SP_Period.objects.filter(period=next_period).exclude(scientific_program__in=auto_validated_sp).filter(status=SP_Period.STATUSES_DRAFT)
  180 + self.change_sp_status(next_sp,SP_Period.STATUSES_SUBMITTED)
  181 + self.send_mail_to_tac_for_evaluation(tac_users,next_period)
  182 +
  183 + # for auto validated sp, we have to change their status
  184 + self.change_sp_status(auto_validated_sp_periods,SP_Period.STATUSES_ACCEPTED)
  185 + for sp in auto_validated_sp_periods:
  186 + sp.is_valid = SP_Period.IS_VALID_ACCEPTED
  187 + sp.save()
  188 +
  189 + SP_PeriodWorkflow.objects.create(period=self.period,action=SP_PeriodWorkflow.EVALUATION)
  190 + if not SP_PeriodWorkflow.objects.filter(action=SP_PeriodWorkflow.VALIDATION, period=self.period).exists():
  191 + print("routine automatic period workflow VALIDATION")
  192 + if next_period.unit_pi_validation_start_date == today :
  193 + next_sp = SP_Period.objects.filter(period=next_period).exclude(scientific_program__in=auto_validated_sp).filter(status=SP_Period.STATUSES_SUBMITTED)
  194 + self.change_sp_status(next_sp,SP_Period.STATUSES_EVALUATED)
  195 + next_sp = SP_Period.objects.filter(period=next_period).exclude(scientific_program__in=auto_validated_sp).filter(status=SP_Period.STATUSES_EVALUATED)
  196 + SP_PeriodWorkflow.objects.create(period=self.period,action=SP_PeriodWorkflow.VALIDATION)
  197 + if not SP_PeriodWorkflow.objects.filter(action=SP_PeriodWorkflow.NOTIFICATION, period=self.period).exists():
  198 + print("routine automatic period workflow NOTIFICATION")
  199 + if next_period in Period.objects.notification_periods():
  200 + next_sp_accepted = SP_Period.objects.filter(period=next_period).filter(is_valid=SP_Period.IS_VALID_ACCEPTED)
  201 + self.change_sp_status(next_sp_accepted,SP_Period.STATUSES_ACCEPTED)
  202 + next_sp_rejected = SP_Period.objects.filter(period=next_period).filter(is_valid=SP_Period.IS_VALID_REJECTED)
  203 + self.change_sp_status(next_sp_rejected,SP_Period.STATUSES_REJECTED)
  204 + next_sp_to_be_notified = next_sp.filter(status=SP_Period.STATUSES_ACCEPTED,is_valid = True)
  205 + self.send_mail_to_observers_for_notification(next_sp_to_be_notified)
  206 + SP_PeriodWorkflow.objects.create(period=self.period,action=SP_PeriodWorkflow.NOTIFICATION)
  207 +
  208 + def routine_process_body(self):
  209 + print("routine automatic period workflow")
  210 + print(SP_PeriodWorkflow.objects.all())
  211 + print(PyrosUser.objects.all())
  212 + for sp_period_workflow in SP_PeriodWorkflow.objects.all():
  213 + print(sp_period_workflow.period)
  214 + print(sp_period_workflow.action)
  215 + self.automatic_period_workflow()
  216 +
  217 +
  218 +if __name__ == "__main__":
  219 +
  220 + # with thread
  221 + RUN_IN_THREAD=True
  222 + # with process
  223 + #RUN_IN_THREAD=False
  224 + print("ARGV OF AGENT SP :",sys.argv)
  225 + if len(sys.argv) > 1 and sys.argv[1] == "test":
  226 + print("i'm in test")
  227 + agentSP = AgentSP(use_db_test=True)
  228 + agentSP.run()
  229 + #agent = build_agent(agentSP, RUN_IN_THREAD=True)
  230 + else:
  231 + agent = build_agent(AgentSP, RUN_IN_THREAD=RUN_IN_THREAD)
  232 + '''
  233 + TEST_MODE, configfile = extract_parameters()
  234 + agent = AgentM("AgentM", configfile, RUN_IN_THREAD)
  235 + agent.setSimulatorMode(TEST_MODE)
  236 + '''
  237 + print(agent)
  238 + agent.run()
... ...
privatedev/plugin/agent/AgentScheduler.py 0 → 100755
  Diff comments   View file @0531624
... ... @@ -0,0 +1,134 @@
  1 +#!/usr/bin/env python3
  2 +
  3 +import sys
  4 +##import utils.Logger as L
  5 +#import threading #, multiprocessing, os
  6 +import time
  7 +
  8 +#from django.db import transaction
  9 +#from common.models import Command
  10 +
  11 +sys.path.append("..")
  12 +sys.path.append("../../../..")
  13 +from src.core.pyros_django.agent.Agent import Agent, build_agent, log
  14 +
  15 +# PM 20190416 recycle code
  16 +#from common.models import *
  17 +from common.models import Sequence
  18 +
  19 +##log = L.setupLogger("AgentXTaskLogger", "AgentX")
  20 +
  21 +
  22 +
  23 +class AgentScheduler(Agent):
  24 +
  25 +
  26 + # FOR TEST ONLY
  27 + # Run this agent in simulator mode
  28 + TEST_MODE = False
  29 + # Run the assertion tests at the end
  30 + TEST_WITH_FINAL_TEST = True
  31 + TEST_MAX_DURATION_SEC = None
  32 + #TEST_MAX_DURATION_SEC = 120
  33 +
  34 + # PM 20190416 fucking config path starting: /home/patrick/Dev/PYROS/start_agent.py agentM
  35 + ##_path_data = 'config'
  36 + _path_data = 'config/old_config'
  37 +
  38 + log.debug("PLC instanciated")
  39 +
  40 +
  41 +
  42 + '''
  43 + # Who should I send commands to ?
  44 + #TEST_COMMANDS_DEST = "myself"
  45 + TEST_COMMANDS_DEST = "AgentA"
  46 + # Scenario to be executed
  47 + TEST_COMMANDS_LIST = [
  48 + "go_active",
  49 + "go_idle",
  50 + "go_active",
  51 + "go_idle",
  52 + "go_active",
  53 + "go_idle",
  54 + "exit",
  55 + ]
  56 + '''
  57 +
  58 + """
  59 + =================================================================
  60 + FUNCTIONS RUN INSIDE MAIN THREAD
  61 + =================================================================
  62 + """
  63 + # old config
  64 + # @override
  65 + #def __init__(self, name:str=None, config_filename=None, RUN_IN_THREAD=True):
  66 + # def __init__(self, config_filename=None, RUN_IN_THREAD=True):
  67 + # ##if name is None: name = self.__class__.__name__
  68 + # super().__init__(config_filename, RUN_IN_THREAD)
  69 +
  70 + # new config (obsconfig)
  71 + def __init__(self, name:str=None, RUN_IN_THREAD=True):
  72 + if name is None:
  73 + name = self.__class__.__name__
  74 + super().__init__(RUN_IN_THREAD)
  75 + # @override
  76 + def init(self):
  77 + super().init()
  78 + log.debug("end init()")
  79 + # --- Set the mode according the startmode value
  80 + ##agent_alias = self.__class__.__name__
  81 + ##self.set_mode_from_config(agent_alias)
  82 +
  83 + '''
  84 + # @override
  85 + def load_config(self):
  86 + super().load_config()
  87 + '''
  88 +
  89 + '''
  90 + # @override
  91 + def update_survey(self):
  92 + super().update_survey()
  93 + '''
  94 +
  95 + '''
  96 + # @override
  97 + def get_next_command(self):
  98 + return super().get_next_command()
  99 + '''
  100 +
  101 + # @override
  102 + def do_log(self):
  103 + super().do_log()
  104 +
  105 + def replan_sequences(self):
  106 + print("\n start of sequences (re-)planning...\n")
  107 + time.sleep(5)
  108 + sequences = Sequence.objects.filter(status="TBP")
  109 + print("List of sequences to be planned :")
  110 + for seq in sequences:
  111 + print('-', seq.name, '('+seq.status+')')
  112 + #print('-- with albums : ', seq.albums)
  113 + print("\n ...end of sequences (re-)planning\n")
  114 +
  115 + # Note : called by _routine_process() in Agent
  116 + # @override
  117 + def routine_process_body(self):
  118 + print("The Observatory configuration :")
  119 + self.show_config()
  120 + log.debug("in routine_process_body()")
  121 + self.replan_sequences()
  122 +
  123 + '''
  124 + # @override
  125 + def exec_specific_cmd_end(self, cmd:Command, from_thread=True):
  126 + super().exec_specific_cmd_end(cmd, from_thread)
  127 + '''
  128 +
  129 +
  130 +if __name__ == "__main__":
  131 +
  132 + agent = build_agent(AgentScheduler)
  133 + print(agent)
  134 + agent.run()
... ...
privatedev/plugin/agent_devices/AgentM.py 0 → 100755
  Diff comments   View file @0531624
... ... @@ -0,0 +1,275 @@
  1 +#!/usr/bin/env python3
  2 +
  3 +import sys
  4 +##import utils.Logger as L
  5 +#import threading #, multiprocessing, os
  6 +#import time
  7 +
  8 +#from django.db import transaction
  9 +#from common.models import Command
  10 +sys.path.append("..")
  11 +sys.path.append("../../../..")
  12 +from src.core.pyros_django.agent.Agent import Agent, build_agent, log
  13 +
  14 +# PM 20190416 recycle code
  15 +from src.core.pyros_django.devices.PLC import PLCController
  16 +from src.core.pyros_django.monitoring.plc_checker import PlcChecker
  17 +from common.models import *
  18 +from config.pyros.config_pyros import ConfigPyros
  19 +
  20 +
  21 +
  22 +##log = L.setupLogger("AgentXTaskLogger", "AgentX")
  23 +
  24 +
  25 +
  26 +class AgentM(Agent):
  27 +
  28 +
  29 + # FOR TEST ONLY
  30 + # Run this agent in simulator mode
  31 + TEST_MODE = False
  32 + # Run the assertion tests at the end
  33 + TEST_WITH_FINAL_TEST = True
  34 + TEST_MAX_DURATION_SEC = None
  35 + #TEST_MAX_DURATION_SEC = 120
  36 +
  37 + # PM 20190416 fucking config path starting: /home/patrick/Dev/PYROS/start_agent.py agentM
  38 + ##_path_data = 'config'
  39 + _path_data = 'config/old_config'
  40 + # PM 20190416 recycle code
  41 + plcController = PLCController()
  42 + print ("AGENT ENV: config PLC is (ip={}, port={})".format(plcController.ip, plcController.port))
  43 + plc_checker = PlcChecker()
  44 +
  45 + log.debug("PLC instanciated")
  46 + time_history_minutes = 4
  47 +
  48 +
  49 + '''
  50 + # Who should I send commands to ?
  51 + #TEST_COMMANDS_DEST = "myself"
  52 + TEST_COMMANDS_DEST = "AgentA"
  53 + # Scenario to be executed
  54 + TEST_COMMANDS_LIST = [
  55 + "go_active",
  56 + "go_idle",
  57 + "go_active",
  58 + "go_idle",
  59 + "go_active",
  60 + "go_idle",
  61 + "exit",
  62 + ]
  63 + '''
  64 +
  65 + """
  66 + =================================================================
  67 + FUNCTIONS RUN INSIDE MAIN THREAD
  68 + =================================================================
  69 + """
  70 + # old config
  71 + # @override
  72 + #def __init__(self, name:str=None, config_filename=None, RUN_IN_THREAD=True):
  73 + # def __init__(self, config_filename=None, RUN_IN_THREAD=True):
  74 + # ##if name is None: name = self.__class__.__name__
  75 + # super().__init__(config_filename, RUN_IN_THREAD)
  76 +
  77 + # new config (obsconfig)
  78 + def __init__(self, name:str=None, RUN_IN_THREAD=True):
  79 + if name is None:
  80 + name = self.__class__.__name__
  81 + super().__init__(RUN_IN_THREAD)
  82 + PYROS_CONFIG_FILE = os.environ.get("pyros_config_file")
  83 + if PYROS_CONFIG_FILE:
  84 + CONFIG_PYROS = ConfigPyros(PYROS_CONFIG_FILE).pyros_config
  85 + self.time_history_minutes = int(CONFIG_PYROS.get("ENV").get("time_history"))
  86 + log.info(f"time_history_minutes set to {int(self.time_history_minutes)}")
  87 + # @override
  88 + def init(self):
  89 + super().init()
  90 +
  91 + log.debug("end init()")
  92 + # --- Set the mode according the startmode value
  93 + ##agent_alias = self.__class__.__name__
  94 + ##self.set_mode_from_config(agent_alias)
  95 +
  96 + '''
  97 + # @override
  98 + def load_config(self):
  99 + super().load_config()
  100 + '''
  101 +
  102 + '''
  103 + # @override
  104 + def update_survey(self):
  105 + super().update_survey()
  106 + '''
  107 +
  108 + '''
  109 + # @override
  110 + def get_next_command(self):
  111 + return super().get_next_command()
  112 + '''
  113 +
  114 + # @override
  115 + def do_log(self):
  116 + super().do_log()
  117 +
  118 +
  119 +
  120 + """
  121 + =================================================================
  122 + FUNCTIONS RUN INSIDE A SUB-THREAD (OR A PROCESS) (thread_*())
  123 + =================================================================
  124 + """
  125 +
  126 + # Define your own command step(s) here
  127 + def cmd_step1(self, step:int):
  128 + cmd = self._current_specific_cmd
  129 + cmd.result = f"in step #{step}/{self._thread_total_steps_number}"
  130 + cmd.save()
  131 + """
  132 + if self.RUN_IN_THREAD:
  133 + print("(save from thread)")
  134 + cmd.save()
  135 + else:
  136 + #@transaction.atomic
  137 + print("(save from process)")
  138 + with transaction.atomic():
  139 + cmd.save()
  140 + #Command.objects.select_for_update()
  141 + """
  142 +
  143 + def cmd_step2(self, step:int):
  144 + self.cmd_step1(step)
  145 + def cmd_step3(self, step:int):
  146 + self.cmd_step1(step)
  147 + def cmd_step4(self, step:int):
  148 + self.cmd_step1(step)
  149 +
  150 + """
  151 + # @override
  152 + def thread_exec_specific_cmd_step(self, step:int, sleep_time:float=1.0):
  153 + self.thread_stop_if_asked()
  154 + cmd = self._current_specific_cmd
  155 + print(f">>>>> Thread (cmd {cmd.name}): step #{step}/5")
  156 + self.sleep(sleep_time)
  157 + """
  158 +
  159 + '''
  160 + # @override
  161 + def exec_specific_cmd_start(self, cmd:Command, from_thread=True):
  162 + super().exec_specific_cmd_start(cmd, from_thread)
  163 + '''
  164 +
  165 + # @override
  166 + # previous name of function : routine_process
  167 + # Note : in Agent.py, routine_process_body seems to be the main function of routine of the agent
  168 + # We need to override routine_process_body and not routine_process
  169 + def routine_process_body(self):
  170 + log.debug("in routine_process_body()")
  171 + print("TODO: we recycle code")
  172 + status_plc = self.plcController.getStatus()
  173 + if self.parseNewStatus(status_plc):
  174 + self.saveWeather()
  175 + #self.saveInternalMonitoring()
  176 +
  177 + def parseNewStatus(self,status_plc):
  178 + # """ PM 20181009 parse new status for config
  179 + # Find return string "plc_status" positin within status_plc
  180 + if status_plc.find('PLC_STATUS') >= 0:
  181 + self.plc_checker.chk_config(status_plc)
  182 + return True
  183 + return False
  184 +
  185 + def saveWeather(self):
  186 + outside = WeatherWatch()
  187 + inside = SiteWatch()
  188 + datetimenow = datetime.now(timezone.utc)
  189 + latest_entry_of_history = WeatherWatchHistory.objects.all().order_by("-datetime").first()
  190 + if latest_entry_of_history != None:
  191 + # Get last entry of WeatherWatchHistory as WeatherWatch
  192 + latest_entry_of_history_as_weather = WeatherWatch.objects.get(id=latest_entry_of_history.weather.id)
  193 + outside_attributes_values = {}
  194 + for sensor in self.plc_checker.monitoring_names.keys():
  195 + if sensor in self.plc_checker.inside_sensors:
  196 + value = self.plc_checker.get_sensor(sensor)
  197 + inside.setAttribute(sensor,value)
  198 + else:
  199 + value = self.plc_checker.get_sensor(sensor)
  200 + outside.setAttribute(sensor, value)
  201 + outside_attributes_values[sensor] = value
  202 + outside.setGlobalStatus()
  203 + outside.save()
  204 + #inside.save()
  205 + # We don't have an history for weatherwatch
  206 + if latest_entry_of_history == None:
  207 + weather_history = WeatherWatchHistory()
  208 + weather_history.weather = outside
  209 + for sensor in outside_attributes_values.keys():
  210 + weather_history.setAttribute(sensor,outside_attributes_values.get(sensor))
  211 + # save also sensors
  212 + weather_history.save()
  213 + else:
  214 + time_between_history_and_latest_entry = datetimenow - latest_entry_of_history_as_weather.updated
  215 + sec_diff = time_between_history_and_latest_entry.total_seconds() / 60
  216 + # if diff between last entry of history and current time if greather than x then we save a new entry in history
  217 + if int(sec_diff) > self.time_history_minutes:
  218 + weather_history = WeatherWatchHistory()
  219 + weather_history.weather = outside
  220 + for sensor in outside_attributes_values.keys():
  221 + weather_history.setAttribute(sensor,outside_attributes_values.get(sensor))
  222 + weather_history.save()
  223 + log.debug("saved weather")
  224 +
  225 + def isInsideMonitoring(self,key):
  226 + print(key)
  227 + if key in ("Power_input","Roof_state"):
  228 + return True
  229 + else:
  230 + return False
  231 + # @override
  232 + def thread_exec_specific_cmd_main(self):
  233 + # This is optional
  234 + self.thread_set_total_steps_number(5)
  235 +
  236 + # HERE, write your own scenario
  237 +
  238 + # scenario OK
  239 + self.thread_exec_specific_cmd_step(1, self.cmd_step1, 1)
  240 + self.thread_exec_specific_cmd_step(2, self.cmd_step2, 3)
  241 + self.thread_exec_specific_cmd_step(3, self.cmd_step1, 5)
  242 + self.thread_exec_specific_cmd_step(4, self.cmd_step3, 10)
  243 + self.thread_exec_specific_cmd_step(5, self.cmd_step1, 4)
  244 + # ... as many as you need
  245 +
  246 + """ autre scenario
  247 + self.thread_exec_specific_cmd_step(1, self.cmd_step1, 1)
  248 + self.thread_exec_specific_cmd_step(2, self.cmd_step2, 2)
  249 + self.thread_exec_specific_cmd_step(3, self.cmd_step1, 2)
  250 + self.thread_exec_specific_cmd_step(4, self.cmd_step3, 2)
  251 + self.thread_exec_specific_cmd_step(5, self.cmd_step1, 3)
  252 + """
  253 +
  254 + '''
  255 + # @override
  256 + def exec_specific_cmd_end(self, cmd:Command, from_thread=True):
  257 + super().exec_specific_cmd_end(cmd, from_thread)
  258 + '''
  259 +
  260 +
  261 +if __name__ == "__main__":
  262 +
  263 + # with thread
  264 + RUN_IN_THREAD=True
  265 + # with process
  266 + #RUN_IN_THREAD=False
  267 +
  268 + agent = build_agent(AgentM, RUN_IN_THREAD=RUN_IN_THREAD)
  269 + '''
  270 + TEST_MODE, configfile = extract_parameters()
  271 + agent = AgentM("AgentM", configfile, RUN_IN_THREAD)
  272 + agent.setSimulatorMode(TEST_MODE)
  273 + '''
  274 + print(agent)
  275 + agent.run()
... ...
pyros.py
  Diff comments   View file @0531624
... ... @@ -63,6 +63,7 @@ AGENTS = {
63 63 "agentA": "AgentA",
64 64 "agentB": "AgentB",
65 65 "agentC": "AgentC",
  66 + "agentSST": "agent",
66 67 # "agentDevice" : "AgentDevice",
67 68 # "agentDeviceTelescopeGemini" : "AgentDeviceTelescopeGemini",
68 69 "agentDeviceGemini": "AgentDeviceGemini",
... ... @@ -96,6 +97,9 @@ if IS_WINDOWS:
96 97 PYTHON = "python.exe"
97 98  
98 99  
  100 +PROJECT_ROOT_PATH = os.getcwd()
  101 +os.environ["PROJECT_ROOT_PATH"] = PROJECT_ROOT_PATH
  102 +
99 103 try:
100 104 # WITH_DOCKER is an environment varialbe from our Docker image
101 105 WITH_DOCKER = os.environ['WITH_DOCKER']
... ... @@ -783,10 +787,11 @@ def initdb():
783 787 @click.option('--configfile', '-c', help='the configuration file to be used')
784 788 @click.option('--observatory', '-o', help='the observatory name to be used')
785 789 @click.option('--unit', '-u', help='the unit name to be used')
  790 +@click.option("--computer_hostname","-cp", help="The name of simulated computer hostname")
786 791 # @click.option('--format', '-f', type=click.Choice(['html', 'xml', 'text']), default='html', show_default=True)
787 792 # @click.option('--port', default=8000)
788 793 # def start(agent:str, configfile:str, test, verbosity):
789   -def start(agent: str, configfile: str, observatory: str, unit: str):
  794 +def start(agent: str, configfile: str, observatory: str, unit: str, computer_hostname: str):
790 795 log.debug("Running start command")
791 796 try:
792 797 from config.pyros.config_pyros import ConfigPyros
... ... @@ -923,7 +928,8 @@ def start(agent: str, configfile: str, observatory: str, unit: str):
923 928 '''
924 929 if agent_name in ["agentM", "agentSP", "agentScheduler"]:
925 930 os.chdir(PYROS_DJANGO_BASE_DIR + os.sep + agent_folder)
926   -
  931 + else:
  932 + os.chdir(PYROS_DJANGO_BASE_DIR+"/agent/")
927 933 # cmd = "-m AgentX"
928 934 # cmd = f" Agent{agent_name[5:]}.py {configfile}"
929 935 cmd = f"Agent{agent_name[5:]}.py"
... ... @@ -937,6 +943,8 @@ def start(agent: str, configfile: str, observatory: str, unit: str):
937 943 cmd += " -v"
938 944 if configfile:
939 945 cmd += " {configfile}"
  946 + if computer_hostname:
  947 + cmd += " -c {computer_hostname}"
940 948  
941 949 # if not test_mode(): current_processes.append( [execProcessFromVenvAsync(cmd), agent_name, -1] )
942 950 # Append this process ( [process id, agent_name, result=failure] )
... ...
src/core/pyros_django/agent/Agent.py
  Diff comments   View file @0531624
... ... @@ -1854,7 +1854,13 @@ def build_agent(Agent_type:Agent, RUN_IN_THREAD=True):
1854 1854 return agent
1855 1855 #agent = Agent_type(name, configfile, RUN_IN_THREAD)
1856 1856 # Get the information of the agent name (name of class) within obsconfig and get the "is_real" attribute
1857   - WITH_SIM = not agent.get_config().get_agent_information(agent.unit,agent.name)["is_real"]
  1857 + if agent.name in agent.get_config().get_agents(agent.unit).keys():
  1858 + if agent.get_config().get_agent_information(agent.unit,agent.name).get("is_real"):
  1859 + WITH_SIM = not agent.get_config().get_agent_information(agent.unit,agent.name)["is_real"]
  1860 + else:
  1861 + WITH_SIM = True
  1862 + else:
  1863 + WITH_SIM = True
1858 1864 agent._set_with_simulator(WITH_SIM)
1859 1865 agent._set_test_mode(TEST_MODE)
1860 1866 #print(logger)
... ...
src/core/pyros_django/agent/AgentSST.py 0 → 100644
  Diff comments   View file @0531624
... ... @@ -0,0 +1,122 @@
  1 +#!/usr/bin/env python3
  2 +
  3 +from pathlib import Path
  4 +import subprocess
  5 +import sys, os
  6 +##import utils.Logger as L
  7 +#import threading #, multiprocessing, os
  8 +import time
  9 +
  10 +#from django.db import transaction
  11 +#from common.models import Command
  12 +
  13 +from Agent import Agent, build_agent, log
  14 +import socket
  15 +
  16 +
  17 +class AgentSST(Agent):
  18 + computer = "MainComputer"
  19 + _previous_dir = ""
  20 + PROJECT_ROOT_PATH = ""
  21 + VENV_PYTHON = ""
  22 + subprocess_dict = {}
  23 +
  24 +
  25 + def __init__(self, name:str=None, RUN_IN_THREAD=True,sim_computer=None):
  26 +
  27 + super().__init__(RUN_IN_THREAD)
  28 + self.PROJECT_ROOT_PATH = os.environ["PROJECT_ROOT_PATH"]
  29 + if name is None:
  30 + name = self.__class__.__name__
  31 + # if sim_computer:
  32 + # self.computer = sim_computer
  33 + # else:
  34 + # self.computer = socket.gethostname()
  35 + WITH_DOCKER = False
  36 + if os.environ.get("WITH_DOCKER"):
  37 + WITH_DOCKER = True
  38 + if WITH_DOCKER:
  39 + VENV_ROOT = ""
  40 + VENV = ""
  41 + VENV_BIN = ""
  42 + else:
  43 + VENV_ROOT = "venv"
  44 + VENV = "venv_py3_pyros"
  45 + VENV_BIN = (
  46 + self.PROJECT_ROOT_PATH
  47 + + os.sep + VENV_ROOT
  48 + + os.sep + VENV
  49 + + os.sep + "bin"
  50 + + os.sep
  51 + )
  52 + VENV_PYTHON = VENV_BIN + "python3"
  53 + log.info(f"PC hostname is {self.computer}")
  54 + self.start_agents()
  55 +
  56 + def start_agents(self,agent_name=None):
  57 + """
  58 + Start all agents or one agent of obs_config
  59 +
  60 + Args:
  61 + agent_name (_type_, optional): Specific agent name to start. Defaults to None.
  62 + """
  63 + obs_config = self.get_config()
  64 +
  65 + agents = obs_config.get_agents_per_computer(obs_config.unit_name).get(self.computer)
  66 + if agents is None:
  67 + print("Computer not found in obs config")
  68 + exit(1)
  69 + #self.change_dir(self.PROJECT_ROOT_PATH)
  70 + if agent_name:
  71 + agent = agent_name
  72 + # Start a specific agent of obs_config (restart)
  73 + agent_informations = obs_config.get_agent_information(obs_config.unit_name,agent)
  74 + protocol = agent_informations.get("protocol")
  75 + if protocol:
  76 + protocol_folder_abs_path = os.path.join(self.PROJECT_ROOT_PATH, os.path.dirname(protocol))
  77 +
  78 + protocol_script_name = protocol.split("/")[-1]
  79 + if os.path.exists(protocol_folder_abs_path + os.sep + protocol_script_name):
  80 + cmd = self.VENV_PYTHON + protocol_folder_abs_path + os.sep + protocol_script_name
  81 +
  82 + process = subprocess.Popen(f"{cmd}", shell=True, stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT)
  83 + self.subprocess_dict[agent] = process
  84 + log.info(f"Start agent {agent} with cmd {cmd}")
  85 +
  86 + else:
  87 + # Start every agent of obs_config (initial start)
  88 + for agent in agents:
  89 + agent_informations = obs_config.get_agent_information(obs_config.unit_name,agent)
  90 + protocol = agent_informations.get("protocol")
  91 + if protocol:
  92 + protocol_folder_abs_path = os.path.join(self.PROJECT_ROOT_PATH, os.path.dirname(protocol))
  93 +
  94 + protocol_script_name = protocol.split("/")[-1]
  95 + if os.path.exists(protocol_folder_abs_path + os.sep + protocol_script_name):
  96 + cmd = self.VENV_PYTHON + protocol_folder_abs_path + os.sep + protocol_script_name
  97 +
  98 + process = subprocess.Popen(f"{cmd}", shell=True, stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT)
  99 + self.subprocess_dict[agent] = process
  100 + log.info(f"Start agent {agent} with cmd {cmd}")
  101 +
  102 + def start_agent(self, agent_name:str):
  103 + """
  104 + Start a specific agent of obs_config (Restart)
  105 +
  106 + Args:
  107 + agent_name (str): Name of agent to start
  108 + """
  109 + self.start_agents(agent_name)
  110 +
  111 + def routine_process_body(self):
  112 + for process in self.subprocess_dict.values():
  113 + process.terminate()
  114 + process.wait()
  115 + exit(0)
  116 + # needs to receive new commands to stop or start new agents
  117 + #start_agents(self)
  118 +
  119 +if __name__ == "__main__":
  120 +
  121 + agent = build_agent(AgentSST)
  122 + agent.run()
0 123 \ No newline at end of file
... ...
src/core/pyros_django/dashboard/views.py
  Diff comments   View file @0531624
... ... @@ -211,7 +211,6 @@ def get_lastest_and_last_x_minutes_before_latest_weather(x:float=0):
211 211 # If we have entries in WeatherWatchHistory
212 212 if WeatherWatchHistory.objects.all().exists():
213 213 # Get the first entry in history that is less or equal than time_start_range
214   - # TODO : améliorer la récupération du first_weather_of_time_start -> peut ne pas respecter le temps - x donné
215 214 first_weather_of_time_start = WeatherWatch.objects.filter(updated__lte=time_start_range).order_by("-updated").first()
216 215 if first_weather_of_time_start != None:
217 216 first_weather_of_time_start_id = first_weather_of_time_start.id
... ...