Commit a78870200222c0c5d4c9afeba733dae7f13182d4

Authored by Etienne Pallier
1 parent 7d179978
Exists in dev

GROSSE OPTIM : plus besoin du script intermédiaire "start_agent.py" !!!

- pyros.py lance directement "cd src/agent/ ; python AgentX.py"
- Multi-agents : AgentA et AgentX fonctionnent en parallèle
- Mode opératoire pour lancer un agent:
		- ./pyros.py start agentX [-c configfile]
		(ou encore: activer l'environnement virtuel, puis lancer "./AgentX.py
configfile")
		- pour démarrer agentA : ouvrir un autre terminal et taper "./pyros.py
start agentA"
		- pour utiliser thread ou processus : il suffit de mettre la constante
RUN_IN_THREAD de AgentX (ou AgentA) à False ou True
@@ -67,21 +67,22 @@ This software has been tested and validated with the following configurations : @@ -67,21 +67,22 @@ This software has been tested and validated with the following configurations :
67 -------------------------------------------------------------------------------------------- 67 --------------------------------------------------------------------------------------------
68 ## LAST VERSION 68 ## LAST VERSION
69 69
70 -Date: 12/03/2019 70 +Date: 15/03/2019
71 71
72 Author: E. Pallier 72 Author: E. Pallier
73 73
74 -VERSION: 0.20.16 74 +VERSION: 0.20.19
75 75
76 Comment: 76 Comment:
77 - Simplification du mode processus (presque aussi simple que thread) 77 + GROSSE OPTIMISATION : plus besoin du script intermédiaire "start_agent.py" !!!
78 78
79 - Bugfixes & New test  
80 - - Bugfix specific_process() en mode "processus" => meilleure gestion accès concurrent à la BD  
81 - - Bugfix Agent => fonctionne aussi en mode SIMU OFF  
82 - - Ajout d'un gros test à la fin pour s'assurer que les résultats sont conformes à l'attendu (only en mode simu)  
83 -  
84 - - Rappel : il suffit de mettre la constante RUN_IN_THREAD de AgentX à False ou True 79 + - pyros.py lance directement "cd src/agent/ ; python AgentX.py"
  80 + - Multi-agents : AgentA et AgentX fonctionnent en parallèle
  81 + - Mode opératoire pour lancer un agent:
  82 + - ./pyros.py start agentX [-c configfile]
  83 + (ou encore: activer l'environnement virtuel, puis lancer "./AgentX.py configfile")
  84 + - pour démarrer agentA : ouvrir un autre terminal et taper "./pyros.py start agentA"
  85 + - pour utiliser thread ou processus : il suffit de mettre la constante RUN_IN_THREAD de AgentX (ou AgentA) à False ou True
85 86
86 -------------------------------------------------------------------------------------------- 87 --------------------------------------------------------------------------------------------
87 - TECHNICAL DOC: tinyurl.com/pyros-doc 88 - TECHNICAL DOC: tinyurl.com/pyros-doc
@@ -23,9 +23,9 @@ DEBUG = True @@ -23,9 +23,9 @@ DEBUG = True
23 INIT_FIXTURE = "initial_fixture.json" 23 INIT_FIXTURE = "initial_fixture.json"
24 24
25 AGENTS = { 25 AGENTS = {
26 - #"agentX" : "majordome",  
27 - "agentX" : "agent",  
28 - "agentA" : "agent", 26 + #"agentX" : "agent",
  27 + "agentX" : "AgentX",
  28 + "agentA" : "AgentA",
29 "webserver" : "webserver", 29 "webserver" : "webserver",
30 "monitoring" : "monitoring", 30 "monitoring" : "monitoring",
31 "majordome" : "majordome", 31 "majordome" : "majordome",
@@ -350,11 +350,24 @@ def start(agent:str, configfile:str): @@ -350,11 +350,24 @@ def start(agent:str, configfile:str):
350 print("Launching agent", agent_name, "...") 350 print("Launching agent", agent_name, "...")
351 #if not test_mode(): execProcess(VENV_BIN + " manage.py runserver") 351 #if not test_mode(): execProcess(VENV_BIN + " manage.py runserver")
352 #if not test_mode(): execProcessFromVenv("start_agent_" + agent_name + ".py " + configfile) 352 #if not test_mode(): execProcessFromVenv("start_agent_" + agent_name + ".py " + configfile)
  353 +
  354 + # OLD format agents: majordome, monitoring, alert...
353 cmd = "start_agent.py " + agent_name + " " + configfile 355 cmd = "start_agent.py " + agent_name + " " + configfile
  356 +
  357 + # Agent "webserver"
354 if agent_name == "webserver": 358 if agent_name == "webserver":
355 cmd = "manage.py runserver" 359 cmd = "manage.py runserver"
356 os.chdir("src") 360 os.chdir("src")
357 #if not test_mode(): execProcessFromVenv("start_agent.py " + agent_name + " " + configfile) 361 #if not test_mode(): execProcessFromVenv("start_agent.py " + agent_name + " " + configfile)
  362 +
  363 + # Agent "agentA", "agentB", "agentX", ...
  364 + if agent_name.startswith("agent"):
  365 + # Run agent without actual commands sent to devices (FOR_REAL=False)
  366 + ##agentX.run(FOR_REAL=True)
  367 + os.chdir("src/agent/")
  368 + #cmd = "-m AgentX"
  369 + cmd = f" Agent{agent_name[5:]}.py " + configfile
  370 +
358 if not test_mode(): execProcessFromVenv(cmd) 371 if not test_mode(): execProcessFromVenv(cmd)
359 # self._change_dir("..") 372 # self._change_dir("..")
360 373
src/agent/Agent.py 100644 → 100755
1 -VERSION = "0.5"  
2 -  
3 -  
4 -"""  
5 -=================================================================  
6 - IMPORT PYTHON PACKAGES  
7 -=================================================================  
8 -"""  
9 -  
10 -#from __future__ import absolute_import  
11 -  
12 -import time  
13 -from datetime import datetime, timedelta  
14 -import os  
15 -import threading, multiprocessing  
16 -#import ctypes  
17 -import copy  
18 -from django.db import transaction  
19 -from django import db  
20 - 1 +#!/usr/bin/env python3
21 2
  3 +VERSION = "0.5"
22 4
23 5
24 """TODO: 6 """TODO:
@@ -30,42 +12,85 @@ from django import db @@ -30,42 +12,85 @@ from django import db
30 """ 12 """
31 13
32 14
33 -# from django.core.exceptions import ObjectDoesNotExist  
34 -# from django.db.models import Q  
35 -from django.shortcuts import get_object_or_404  
36 -from django.conf import settings as djangosettings 15 +"""
  16 +=================================================================
  17 + SETUP FOR DJANGO
  18 +=================================================================
  19 +"""
37 20
38 -import utils.Logger as L 21 +import os
  22 +import sys
39 23
40 -from config.configpyros import ConfigPyros 24 +from django.conf import settings as djangosettings
  25 +
  26 +# Conseil sur le net:
  27 +#https://stackoverflow.com/questions/16853649/how-to-execute-a-python-script-from-the-django-shell
  28 +#""
  29 +#import sys, os
  30 +#sys.path.append('/path/to/your/django/app')
  31 +#os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
  32 +#from django.conf import settings
  33 +#""
  34 +
  35 +# To avoid a "ModuleNotFoundError: No module named 'dashboard'"... (not even 1 app found) :
  36 +##sys.path.insert(0, os.path.abspath(".."))
  37 +##sys.path.insert(0, os.path.abspath("src"))
  38 +##sys.path.insert(0, "../src")
  39 +##sys.path.insert(0, "src")
  40 +# To avoid a "ModuleNotFoundError: No module named 'src'"
  41 +sys.path.append("..")
  42 +sys.path.append("../..")
  43 +##sys.path.append("src")
  44 +print("Starting with this sys.path", sys.path)
  45 +
  46 +# DJANGO setup
  47 +# print("file is", __file__)
  48 +# mypath = os.getcwd()
  49 +# Go into src/
  50 +##os.chdir("..")
  51 +##os.chdir("src")
  52 +print("Current directory : " + str(os.getcwd()))
  53 +
  54 +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "src.pyros.settings")
  55 +# os.environ['SECRET_KEY'] = 'abc'
  56 +# os.environ['ENVIRONMENT'] = 'production'
  57 +import django
  58 +
  59 +django.setup()
  60 +
  61 +print("DB2 used is:", djangosettings.DATABASES["default"]["NAME"])
  62 +print()
41 63
42 -"""  
43 -import observation_manager  
44 -import observation_manager.tasks  
45 -import scheduler  
46 -import scheduler.tasks as sched_task  
47 -import monitoring.tasks  
48 -import alert_manager.tasks  
49 -"""  
50 -# from common.models import *  
51 -from common.models import Config, Log, PlcDeviceStatus  
52 -from common.models import AgentSurvey, Command  
53 -from dashboard.views import get_sunelev  
54 -from devices.TelescopeRemoteControlDefault import TelescopeRemoteControlDefault  
55 64
56 """ 65 """
57 -from devices.CameraNIR import NIRCameraController  
58 -from devices.CameraVIS import VISCameraController  
59 -from devices.Dome import DomeController  
60 -from devices.PLC import PLCController  
61 -from devices.Telescope import TelescopeController  
62 -from majordome.MajordomeDecorators import *  
63 -from utils.JDManipulator import * 66 +=================================================================
  67 + IMPORT PYTHON PACKAGES
  68 +=================================================================
64 """ 69 """
65 70
  71 +# --- GENERAL PURPOSE IMPORT ---
  72 +#from __future__ import absolute_import
  73 +import time
  74 +import threading, multiprocessing
66 from threading import Thread 75 from threading import Thread
  76 +import utils.Logger as L
  77 +#import ctypes
  78 +#import copy
67 79
  80 +# --- DJANGO IMPORT ---
  81 +from django.db import transaction
  82 +from django import db
  83 +# from django.core.exceptions import ObjectDoesNotExist
  84 +# from django.db.models import Q
  85 +#from django.shortcuts import get_object_or_404
  86 +#from django.conf import settings as djangosettings
68 87
  88 +# --- SPECIFIC IMPORT ---
  89 +from config.configpyros import ConfigPyros
  90 +from common.models import AgentSurvey, Command
  91 +#from dashboard.views import get_sunelev
  92 +#from devices.TelescopeRemoteControlDefault import TelescopeRemoteControlDefault
  93 +#from utils.JDManipulator import *
69 94
70 95
71 """ 96 """
@@ -165,7 +190,6 @@ class StoppableThreadEvenWhenSleeping(threading.Thread): @@ -165,7 +190,6 @@ class StoppableThreadEvenWhenSleeping(threading.Thread):
165 self._stop_event.wait(nbsec) 190 self._stop_event.wait(nbsec)
166 191
167 192
168 -  
169 """ 193 """
170 ================================================================= 194 =================================================================
171 class Agent 195 class Agent
@@ -197,7 +221,11 @@ class Agent: @@ -197,7 +221,11 @@ class Agent:
197 >>>>> Thread: starting execution of command specific3 221 >>>>> Thread: starting execution of command specific3
198 >>>>> Thread: PID: 2690, Process Name: Process-3, Thread Name: MainThread 222 >>>>> Thread: PID: 2690, Process Name: Process-3, Thread Name: MainThread
199 """ 223 """
  224 + # with thread
200 RUN_IN_THREAD = True 225 RUN_IN_THREAD = True
  226 + # with process
  227 + #RUN_IN_THREAD = False
  228 +
201 _thread_total_steps_number = 1 229 _thread_total_steps_number = 1
202 230
203 # Run for real, otherwise just print messages without really doing anything 231 # Run for real, otherwise just print messages without really doing anything
@@ -238,15 +266,18 @@ class Agent: @@ -238,15 +266,18 @@ class Agent:
238 _iter_num = 1 266 _iter_num = 1
239 267
240 def __init__(self, name:str="Agent", config_filename:str=None, RUN_IN_THREAD=True): 268 def __init__(self, name:str="Agent", config_filename:str=None, RUN_IN_THREAD=True):
  269 + self.name = name
  270 + self.RUN_IN_THREAD = RUN_IN_THREAD
241 self.set_status(self.STATUS_LAUNCH) 271 self.set_status(self.STATUS_LAUNCH)
242 self.set_mode(self.MODE_IDLE) 272 self.set_mode(self.MODE_IDLE)
243 - self.name = name  
244 if not config_filename: 273 if not config_filename:
245 #config_filename = '/PROJECTS/GFT/SOFT/PYROS_SOFT/CURRENT/config/config_unit_simulunit1.xml' 274 #config_filename = '/PROJECTS/GFT/SOFT/PYROS_SOFT/CURRENT/config/config_unit_simulunit1.xml'
246 config_filename = self.DEFAULT_CONFIG_FILE_NAME 275 config_filename = self.DEFAULT_CONFIG_FILE_NAME
247 #config_file_path, _ = os.path.split(config_filename) 276 #config_file_path, _ = os.path.split(config_filename)
  277 + # If config file name is RELATIVE (i.e. without path, just the file name)
  278 + # => give it an absolute path (and remove "src/agent/" from it)
248 if config_filename == os.path.basename(config_filename): 279 if config_filename == os.path.basename(config_filename):
249 - config_filename = os.path.abspath(self.CONFIG_DIR + os.sep + config_filename) 280 + config_filename = os.path.abspath(self.CONFIG_DIR + os.sep + config_filename).replace("src/agent/","")
250 print("Config file used is", config_filename) 281 print("Config file used is", config_filename)
251 #print("current path", os.getcwd()) 282 #print("current path", os.getcwd())
252 #print("this file path :", __file__) 283 #print("this file path :", __file__)
@@ -845,12 +876,41 @@ class Agent: @@ -845,12 +876,41 @@ class Agent:
845 cmd.set_as_running() 876 cmd.set_as_running()
846 """ 877 """
847 878
  879 + # Define your own command step(s) here
  880 + def cmd_step(self, step:int):
  881 + cmd = self._current_specific_cmd
  882 + cmd.result = f"in step #{step}/{self._thread_total_steps_number}"
  883 + cmd.save()
  884 +
848 # Default body of the specific processing 885 # Default body of the specific processing
849 # Should be overriden by subclass 886 # Should be overriden by subclass
850 def thread_exec_specific_cmd_main(self): 887 def thread_exec_specific_cmd_main(self):
  888 + """
851 cmd = self._current_specific_cmd 889 cmd = self._current_specific_cmd
852 print("Doing nothing, just sleeping...") 890 print("Doing nothing, just sleeping...")
853 self.sleep(3) 891 self.sleep(3)
  892 + """
  893 + # This is optional
  894 + self.thread_set_total_steps_number(5)
  895 +
  896 + # HERE, write your own scenario
  897 +
  898 + # scenario OK
  899 + self.thread_exec_specific_cmd_step(1, self.cmd_step, 1)
  900 + self.thread_exec_specific_cmd_step(2, self.cmd_step, 3)
  901 + self.thread_exec_specific_cmd_step(3, self.cmd_step, 5)
  902 + self.thread_exec_specific_cmd_step(4, self.cmd_step, 10)
  903 + self.thread_exec_specific_cmd_step(5, self.cmd_step, 4)
  904 + # ... as many as you need
  905 +
  906 + """ other scenario
  907 + self.thread_exec_specific_cmd_step(1, self.cmd_step1, 1)
  908 + self.thread_exec_specific_cmd_step(2, self.cmd_step2, 2)
  909 + self.thread_exec_specific_cmd_step(3, self.cmd_step1, 2)
  910 + self.thread_exec_specific_cmd_step(4, self.cmd_step3, 2)
  911 + self.thread_exec_specific_cmd_step(5, self.cmd_step1, 3)
  912 + """
  913 +
854 914
855 def thread_exec_specific_cmd_end(self): 915 def thread_exec_specific_cmd_end(self):
856 """ specific command execution tearing down """ 916 """ specific command execution tearing down """
@@ -891,12 +951,10 @@ class Agent: @@ -891,12 +951,10 @@ class Agent:
891 def thread_set_total_steps_number(self, nbsteps): 951 def thread_set_total_steps_number(self, nbsteps):
892 self._thread_total_steps_number = nbsteps 952 self._thread_total_steps_number = nbsteps
893 953
894 -  
895 """ 954 """
896 =================================== 955 ===================================
897 OLD FUNCTIONS TO BE REMOVED 956 OLD FUNCTIONS TO BE REMOVED
898 =================================== 957 ===================================
899 - """  
900 958
901 def _plc_is_not_auto(self): 959 def _plc_is_not_auto(self):
902 if not self.plc_is_connected(): 960 if not self.plc_is_connected():
@@ -918,5 +976,17 @@ class Agent: @@ -918,5 +976,17 @@ class Agent:
918 976
919 def is_night(self): 977 def is_night(self):
920 return get_sunelev() < -10 978 return get_sunelev() < -10
  979 + """
  980 +
  981 +
  982 +if __name__ == "__main__":
  983 +
  984 + configfile = None
921 985
  986 + # arg 1 : config file
  987 + if len(sys.argv) == 2:
  988 + configfile = sys.argv[1]
922 989
  990 + agent = Agent("GenericAgent", configfile, RUN_IN_THREAD=True)
  991 + print(agent)
  992 + agent.run()
src/agent/AgentA.py
  1 +#!/usr/bin/env python3
1 2
2 -import utils.Logger as L  
3 -import threading #, multiprocessing, os  
4 -import time  
5 3
6 -from django.db import transaction  
7 -from common.models import Command 4 +import sys
  5 +##import utils.Logger as L
8 6
9 -from .Agent import Agent 7 +##from .Agent import Agent
  8 +from Agent import Agent
10 9
11 10
12 -  
13 -log = L.setupLogger("AgentXTaskLogger", "AgentX") 11 +##log = L.setupLogger("AgentXTaskLogger", "AgentX")
14 12
15 13
16 14
@@ -40,7 +38,7 @@ class AgentA(Agent): @@ -40,7 +38,7 @@ class AgentA(Agent):
40 >>>>> Thread: PID: 2690, Process Name: Process-3, Thread Name: MainThread 38 >>>>> Thread: PID: 2690, Process Name: Process-3, Thread Name: MainThread
41 """ 39 """
42 RUN_IN_THREAD = True 40 RUN_IN_THREAD = True
43 - RUN_IN_THREAD = False 41 + #RUN_IN_THREAD = False
44 42
45 43
46 """ 44 """
@@ -50,9 +48,9 @@ class AgentA(Agent): @@ -50,9 +48,9 @@ class AgentA(Agent):
50 """ 48 """
51 49
52 # @override 50 # @override
53 - def __init__(self, name:str=None, config_filename=None): 51 + def __init__(self, name:str=None, config_filename=None, RUN_IN_THREAD=True):
54 if name is None: name = self.__class__.__name__ 52 if name is None: name = self.__class__.__name__
55 - super().__init__(name, config_filename, self.RUN_IN_THREAD) 53 + super().__init__(name, config_filename, RUN_IN_THREAD)
56 54
57 # @override 55 # @override
58 def init(self): 56 def init(self):
@@ -159,3 +157,23 @@ class AgentA(Agent): @@ -159,3 +157,23 @@ class AgentA(Agent):
159 def exec_specific_cmd_end(self, cmd:Command, from_thread=True): 157 def exec_specific_cmd_end(self, cmd:Command, from_thread=True):
160 super().exec_specific_cmd_end(cmd, from_thread) 158 super().exec_specific_cmd_end(cmd, from_thread)
161 ''' 159 '''
  160 +
  161 +
  162 +
  163 +if __name__ == "__main__":
  164 +
  165 + # with thread
  166 + RUN_IN_THREAD=True
  167 + # with process
  168 + RUN_IN_THREAD=False
  169 +
  170 + configfile = None
  171 +
  172 + # arg 1 : config file
  173 + if len(sys.argv) == 2:
  174 + configfile = sys.argv[1]
  175 +
  176 + #agent = AgentX()
  177 + agent = AgentA("AgentA", configfile, RUN_IN_THREAD)
  178 + print(agent)
  179 + agent.run()
src/agent/AgentX.py 100644 → 100755
  1 +#!/usr/bin/env python3
1 2
2 -import utils.Logger as L  
3 -import threading #, multiprocessing, os  
4 -import time 3 +import sys
  4 +##import utils.Logger as L
  5 +#import threading #, multiprocessing, os
  6 +#import time
5 7
6 -from django.db import transaction  
7 -from common.models import Command 8 +#from django.db import transaction
  9 +#from common.models import Command
8 10
9 -from .Agent import Agent 11 +##from .Agent import Agent
  12 +from Agent import Agent
10 13
11 14
12 15
13 -log = L.setupLogger("AgentXTaskLogger", "AgentX") 16 +##log = L.setupLogger("AgentXTaskLogger", "AgentX")
14 17
15 18
16 19
17 class AgentX(Agent): 20 class AgentX(Agent):
18 21
19 """ 22 """
20 - How to run this agent thread_exec_specific_cmd() method ?  
21 - - True = inside a Thread (cannot be killed, must be asked to stop, and inadequate for computation)  
22 - - False = inside a Process  
23 - If thread, displays :  
24 - >>>>> Thread: starting execution of command specific1  
25 - >>>>> Thread: PID: 2695, Process Name: MainProcess, Thread Name: Thread-1  
26 - ...  
27 - >>>>> Thread: starting execution of command specific2  
28 - >>>>> Thread: PID: 2695, Process Name: MainProcess, Thread Name: Thread-2  
29 - ...  
30 - >>>>> Thread: starting execution of command specific3  
31 - >>>>> Thread: PID: 2695, Process Name: MainProcess, Thread Name: Thread-3  
32 - If process, displays :  
33 - >>>>> Thread: starting execution of command specific1  
34 - >>>>> Thread: PID: 2687, Process Name: Process-1, Thread Name: MainThread  
35 - ...  
36 - >>>>> Thread: starting execution of command specific2  
37 - >>>>> Thread: PID: 2689, Process Name: Process-2, Thread Name: MainThread  
38 - ...  
39 - >>>>> Thread: starting execution of command specific3  
40 - >>>>> Thread: PID: 2690, Process Name: Process-3, Thread Name: MainThread  
41 - """  
42 - RUN_IN_THREAD = True  
43 - RUN_IN_THREAD = False  
44 -  
45 -  
46 - """  
47 ================================================================= 23 =================================================================
48 FUNCTIONS RUN INSIDE MAIN THREAD 24 FUNCTIONS RUN INSIDE MAIN THREAD
49 ================================================================= 25 =================================================================
50 """ 26 """
51 27
52 # @override 28 # @override
53 - def __init__(self, name:str=None, config_filename=None): 29 + def __init__(self, name:str=None, config_filename=None, RUN_IN_THREAD=True):
54 if name is None: name = self.__class__.__name__ 30 if name is None: name = self.__class__.__name__
55 - super().__init__(name, config_filename, self.RUN_IN_THREAD) 31 + super().__init__(name, config_filename, RUN_IN_THREAD)
56 32
57 # @override 33 # @override
58 def init(self): 34 def init(self):
@@ -159,3 +135,22 @@ class AgentX(Agent): @@ -159,3 +135,22 @@ class AgentX(Agent):
159 def exec_specific_cmd_end(self, cmd:Command, from_thread=True): 135 def exec_specific_cmd_end(self, cmd:Command, from_thread=True):
160 super().exec_specific_cmd_end(cmd, from_thread) 136 super().exec_specific_cmd_end(cmd, from_thread)
161 ''' 137 '''
  138 +
  139 +
  140 +if __name__ == "__main__":
  141 +
  142 + # with thread
  143 + RUN_IN_THREAD=True
  144 + # with process
  145 + RUN_IN_THREAD=False
  146 +
  147 + configfile = None
  148 +
  149 + # arg 1 : config file
  150 + if len(sys.argv) == 2:
  151 + configfile = sys.argv[1]
  152 +
  153 + #agent = AgentX()
  154 + agent = AgentX("AgentX", configfile, RUN_IN_THREAD)
  155 + print(agent)
  156 + agent.run()
start_agent.py
@@ -100,6 +100,12 @@ if agent_name == &quot;agentA&quot;: @@ -100,6 +100,12 @@ if agent_name == &quot;agentA&quot;:
100 agentA.run(FOR_REAL=True) 100 agentA.run(FOR_REAL=True)
101 sys.exit(0) 101 sys.exit(0)
102 102
  103 +"""
  104 +from src.agent.Agent import Agent
  105 +agent = Agent(name="agent", config_filename=configfile)
  106 +agent.run(FOR_REAL=True)
  107 +"""
  108 +
103 # Default agent is AgentX 109 # Default agent is AgentX
104 110
105 from src.agent.AgentX import AgentX 111 from src.agent.AgentX import AgentX