PYROS-IRAP / pyros

Commit a78870200222c0c5d4c9afeba733dae7f13182d4

Authored by Etienne Pallier
1 parent 7d179978
Exists in dev

GROSSE OPTIM : plus besoin du script intermédiaire "start_agent.py" !!! 

- pyros.py lance directement "cd src/agent/ ; python AgentX.py"
- Multi-agents : AgentA et AgentX fonctionnent en parallèle
- Mode opératoire pour lancer un agent:
		- ./pyros.py start agentX [-c configfile]
		(ou encore: activer l'environnement virtuel, puis lancer "./AgentX.py
configfile")
		- pour démarrer agentA : ouvrir un autre terminal et taper "./pyros.py
start agentA"
		- pour utiliser thread ou processus : il suffit de mettre la constante
RUN_IN_THREAD de AgentX (ou AgentA) à False ou True
Showing 6 changed files with 213 additions and 110 deletions   Show diff stats
Inline Side-by-side
README.md
  Diff comments   View file @a788702
... ... @@ -67,21 +67,22 @@ This software has been tested and validated with the following configurations :
67 67 --------------------------------------------------------------------------------------------
68 68 ## LAST VERSION
69 69  
70   -Date: 12/03/2019
  70 +Date: 15/03/2019
71 71  
72 72 Author: E. Pallier
73 73  
74   -VERSION: 0.20.16
  74 +VERSION: 0.20.19
75 75  
76 76 Comment:
77   - Simplification du mode processus (presque aussi simple que thread)
  77 + GROSSE OPTIMISATION : plus besoin du script intermédiaire "start_agent.py" !!!
78 78  
79   - Bugfixes & New test
80   - - Bugfix specific_process() en mode "processus" => meilleure gestion accès concurrent à la BD
81   - - Bugfix Agent => fonctionne aussi en mode SIMU OFF
82   - - Ajout d'un gros test à la fin pour s'assurer que les résultats sont conformes à l'attendu (only en mode simu)
83   -
84   - - Rappel : il suffit de mettre la constante RUN_IN_THREAD de AgentX à False ou True
  79 + - pyros.py lance directement "cd src/agent/ ; python AgentX.py"
  80 + - Multi-agents : AgentA et AgentX fonctionnent en parallèle
  81 + - Mode opératoire pour lancer un agent:
  82 + - ./pyros.py start agentX [-c configfile]
  83 + (ou encore: activer l'environnement virtuel, puis lancer "./AgentX.py configfile")
  84 + - pour démarrer agentA : ouvrir un autre terminal et taper "./pyros.py start agentA"
  85 + - pour utiliser thread ou processus : il suffit de mettre la constante RUN_IN_THREAD de AgentX (ou AgentA) à False ou True
85 86  
86 87 --------------------------------------------------------------------------------------------
87 88 - TECHNICAL DOC: tinyurl.com/pyros-doc
... ...
pyros.py
  Diff comments   View file @a788702
... ... @@ -23,9 +23,9 @@ DEBUG = True
23 23 INIT_FIXTURE = "initial_fixture.json"
24 24  
25 25 AGENTS = {
26   - #"agentX" : "majordome",
27   - "agentX" : "agent",
28   - "agentA" : "agent",
  26 + #"agentX" : "agent",
  27 + "agentX" : "AgentX",
  28 + "agentA" : "AgentA",
29 29 "webserver" : "webserver",
30 30 "monitoring" : "monitoring",
31 31 "majordome" : "majordome",
... ... @@ -350,11 +350,24 @@ def start(agent:str, configfile:str):
350 350 print("Launching agent", agent_name, "...")
351 351 #if not test_mode(): execProcess(VENV_BIN + " manage.py runserver")
352 352 #if not test_mode(): execProcessFromVenv("start_agent_" + agent_name + ".py " + configfile)
  353 +
  354 + # OLD format agents: majordome, monitoring, alert...
353 355 cmd = "start_agent.py " + agent_name + " " + configfile
  356 +
  357 + # Agent "webserver"
354 358 if agent_name == "webserver":
355 359 cmd = "manage.py runserver"
356 360 os.chdir("src")
357 361 #if not test_mode(): execProcessFromVenv("start_agent.py " + agent_name + " " + configfile)
  362 +
  363 + # Agent "agentA", "agentB", "agentX", ...
  364 + if agent_name.startswith("agent"):
  365 + # Run agent without actual commands sent to devices (FOR_REAL=False)
  366 + ##agentX.run(FOR_REAL=True)
  367 + os.chdir("src/agent/")
  368 + #cmd = "-m AgentX"
  369 + cmd = f" Agent{agent_name[5:]}.py " + configfile
  370 +
358 371 if not test_mode(): execProcessFromVenv(cmd)
359 372 # self._change_dir("..")
360 373  
... ...
src/agent/Agent.py 100644 → 100755
  Diff comments   View file @a788702
1   -VERSION = "0.5"
2   -
3   -
4   -"""
5   -=================================================================
6   - IMPORT PYTHON PACKAGES
7   -=================================================================
8   -"""
9   -
10   -#from __future__ import absolute_import
11   -
12   -import time
13   -from datetime import datetime, timedelta
14   -import os
15   -import threading, multiprocessing
16   -#import ctypes
17   -import copy
18   -from django.db import transaction
19   -from django import db
20   -
  1 +#!/usr/bin/env python3
21 2  
  3 +VERSION = "0.5"
22 4  
23 5  
24 6 """TODO:
... ... @@ -30,42 +12,85 @@ from django import db
30 12 """
31 13  
32 14  
33   -# from django.core.exceptions import ObjectDoesNotExist
34   -# from django.db.models import Q
35   -from django.shortcuts import get_object_or_404
36   -from django.conf import settings as djangosettings
  15 +"""
  16 +=================================================================
  17 + SETUP FOR DJANGO
  18 +=================================================================
  19 +"""
37 20  
38   -import utils.Logger as L
  21 +import os
  22 +import sys
39 23  
40   -from config.configpyros import ConfigPyros
  24 +from django.conf import settings as djangosettings
  25 +
  26 +# Conseil sur le net:
  27 +#https://stackoverflow.com/questions/16853649/how-to-execute-a-python-script-from-the-django-shell
  28 +#""
  29 +#import sys, os
  30 +#sys.path.append('/path/to/your/django/app')
  31 +#os.environ['DJANGO_SETTINGS_MODULE'] = 'settings'
  32 +#from django.conf import settings
  33 +#""
  34 +
  35 +# To avoid a "ModuleNotFoundError: No module named 'dashboard'"... (not even 1 app found) :
  36 +##sys.path.insert(0, os.path.abspath(".."))
  37 +##sys.path.insert(0, os.path.abspath("src"))
  38 +##sys.path.insert(0, "../src")
  39 +##sys.path.insert(0, "src")
  40 +# To avoid a "ModuleNotFoundError: No module named 'src'"
  41 +sys.path.append("..")
  42 +sys.path.append("../..")
  43 +##sys.path.append("src")
  44 +print("Starting with this sys.path", sys.path)
  45 +
  46 +# DJANGO setup
  47 +# print("file is", __file__)
  48 +# mypath = os.getcwd()
  49 +# Go into src/
  50 +##os.chdir("..")
  51 +##os.chdir("src")
  52 +print("Current directory : " + str(os.getcwd()))
  53 +
  54 +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "src.pyros.settings")
  55 +# os.environ['SECRET_KEY'] = 'abc'
  56 +# os.environ['ENVIRONMENT'] = 'production'
  57 +import django
  58 +
  59 +django.setup()
  60 +
  61 +print("DB2 used is:", djangosettings.DATABASES["default"]["NAME"])
  62 +print()
41 63  
42   -"""
43   -import observation_manager
44   -import observation_manager.tasks
45   -import scheduler
46   -import scheduler.tasks as sched_task
47   -import monitoring.tasks
48   -import alert_manager.tasks
49   -"""
50   -# from common.models import *
51   -from common.models import Config, Log, PlcDeviceStatus
52   -from common.models import AgentSurvey, Command
53   -from dashboard.views import get_sunelev
54   -from devices.TelescopeRemoteControlDefault import TelescopeRemoteControlDefault
55 64  
56 65 """
57   -from devices.CameraNIR import NIRCameraController
58   -from devices.CameraVIS import VISCameraController
59   -from devices.Dome import DomeController
60   -from devices.PLC import PLCController
61   -from devices.Telescope import TelescopeController
62   -from majordome.MajordomeDecorators import *
63   -from utils.JDManipulator import *
  66 +=================================================================
  67 + IMPORT PYTHON PACKAGES
  68 +=================================================================
64 69 """
65 70  
  71 +# --- GENERAL PURPOSE IMPORT ---
  72 +#from __future__ import absolute_import
  73 +import time
  74 +import threading, multiprocessing
66 75 from threading import Thread
  76 +import utils.Logger as L
  77 +#import ctypes
  78 +#import copy
67 79  
  80 +# --- DJANGO IMPORT ---
  81 +from django.db import transaction
  82 +from django import db
  83 +# from django.core.exceptions import ObjectDoesNotExist
  84 +# from django.db.models import Q
  85 +#from django.shortcuts import get_object_or_404
  86 +#from django.conf import settings as djangosettings
68 87  
  88 +# --- SPECIFIC IMPORT ---
  89 +from config.configpyros import ConfigPyros
  90 +from common.models import AgentSurvey, Command
  91 +#from dashboard.views import get_sunelev
  92 +#from devices.TelescopeRemoteControlDefault import TelescopeRemoteControlDefault
  93 +#from utils.JDManipulator import *
69 94  
70 95  
71 96 """
... ... @@ -165,7 +190,6 @@ class StoppableThreadEvenWhenSleeping(threading.Thread):
165 190 self._stop_event.wait(nbsec)
166 191  
167 192  
168   -
169 193 """
170 194 =================================================================
171 195 class Agent
... ... @@ -197,7 +221,11 @@ class Agent:
197 221 >>>>> Thread: starting execution of command specific3
198 222 >>>>> Thread: PID: 2690, Process Name: Process-3, Thread Name: MainThread
199 223 """
  224 + # with thread
200 225 RUN_IN_THREAD = True
  226 + # with process
  227 + #RUN_IN_THREAD = False
  228 +
201 229 _thread_total_steps_number = 1
202 230  
203 231 # Run for real, otherwise just print messages without really doing anything
... ... @@ -238,15 +266,18 @@ class Agent:
238 266 _iter_num = 1
239 267  
240 268 def __init__(self, name:str="Agent", config_filename:str=None, RUN_IN_THREAD=True):
  269 + self.name = name
  270 + self.RUN_IN_THREAD = RUN_IN_THREAD
241 271 self.set_status(self.STATUS_LAUNCH)
242 272 self.set_mode(self.MODE_IDLE)
243   - self.name = name
244 273 if not config_filename:
245 274 #config_filename = '/PROJECTS/GFT/SOFT/PYROS_SOFT/CURRENT/config/config_unit_simulunit1.xml'
246 275 config_filename = self.DEFAULT_CONFIG_FILE_NAME
247 276 #config_file_path, _ = os.path.split(config_filename)
  277 + # If config file name is RELATIVE (i.e. without path, just the file name)
  278 + # => give it an absolute path (and remove "src/agent/" from it)
248 279 if config_filename == os.path.basename(config_filename):
249   - config_filename = os.path.abspath(self.CONFIG_DIR + os.sep + config_filename)
  280 + config_filename = os.path.abspath(self.CONFIG_DIR + os.sep + config_filename).replace("src/agent/","")
250 281 print("Config file used is", config_filename)
251 282 #print("current path", os.getcwd())
252 283 #print("this file path :", __file__)
... ... @@ -845,12 +876,41 @@ class Agent:
845 876 cmd.set_as_running()
846 877 """
847 878  
  879 + # Define your own command step(s) here
  880 + def cmd_step(self, step:int):
  881 + cmd = self._current_specific_cmd
  882 + cmd.result = f"in step #{step}/{self._thread_total_steps_number}"
  883 + cmd.save()
  884 +
848 885 # Default body of the specific processing
849 886 # Should be overriden by subclass
850 887 def thread_exec_specific_cmd_main(self):
  888 + """
851 889 cmd = self._current_specific_cmd
852 890 print("Doing nothing, just sleeping...")
853 891 self.sleep(3)
  892 + """
  893 + # This is optional
  894 + self.thread_set_total_steps_number(5)
  895 +
  896 + # HERE, write your own scenario
  897 +
  898 + # scenario OK
  899 + self.thread_exec_specific_cmd_step(1, self.cmd_step, 1)
  900 + self.thread_exec_specific_cmd_step(2, self.cmd_step, 3)
  901 + self.thread_exec_specific_cmd_step(3, self.cmd_step, 5)
  902 + self.thread_exec_specific_cmd_step(4, self.cmd_step, 10)
  903 + self.thread_exec_specific_cmd_step(5, self.cmd_step, 4)
  904 + # ... as many as you need
  905 +
  906 + """ other scenario
  907 + self.thread_exec_specific_cmd_step(1, self.cmd_step1, 1)
  908 + self.thread_exec_specific_cmd_step(2, self.cmd_step2, 2)
  909 + self.thread_exec_specific_cmd_step(3, self.cmd_step1, 2)
  910 + self.thread_exec_specific_cmd_step(4, self.cmd_step3, 2)
  911 + self.thread_exec_specific_cmd_step(5, self.cmd_step1, 3)
  912 + """
  913 +
854 914  
855 915 def thread_exec_specific_cmd_end(self):
856 916 """ specific command execution tearing down """
... ... @@ -891,12 +951,10 @@ class Agent:
891 951 def thread_set_total_steps_number(self, nbsteps):
892 952 self._thread_total_steps_number = nbsteps
893 953  
894   -
895 954 """
896 955 ===================================
897 956 OLD FUNCTIONS TO BE REMOVED
898 957 ===================================
899   - """
900 958  
901 959 def _plc_is_not_auto(self):
902 960 if not self.plc_is_connected():
... ... @@ -918,5 +976,17 @@ class Agent:
918 976  
919 977 def is_night(self):
920 978 return get_sunelev() < -10
  979 + """
  980 +
  981 +
  982 +if __name__ == "__main__":
  983 +
  984 + configfile = None
921 985  
  986 + # arg 1 : config file
  987 + if len(sys.argv) == 2:
  988 + configfile = sys.argv[1]
922 989  
  990 + agent = Agent("GenericAgent", configfile, RUN_IN_THREAD=True)
  991 + print(agent)
  992 + agent.run()
... ...
src/agent/AgentA.py
  Diff comments   View file @a788702
  1 +#!/usr/bin/env python3
1 2  
2   -import utils.Logger as L
3   -import threading #, multiprocessing, os
4   -import time
5 3  
6   -from django.db import transaction
7   -from common.models import Command
  4 +import sys
  5 +##import utils.Logger as L
8 6  
9   -from .Agent import Agent
  7 +##from .Agent import Agent
  8 +from Agent import Agent
10 9  
11 10  
12   -
13   -log = L.setupLogger("AgentXTaskLogger", "AgentX")
  11 +##log = L.setupLogger("AgentXTaskLogger", "AgentX")
14 12  
15 13  
16 14  
... ... @@ -40,7 +38,7 @@ class AgentA(Agent):
40 38 >>>>> Thread: PID: 2690, Process Name: Process-3, Thread Name: MainThread
41 39 """
42 40 RUN_IN_THREAD = True
43   - RUN_IN_THREAD = False
  41 + #RUN_IN_THREAD = False
44 42  
45 43  
46 44 """
... ... @@ -50,9 +48,9 @@ class AgentA(Agent):
50 48 """
51 49  
52 50 # @override
53   - def __init__(self, name:str=None, config_filename=None):
  51 + def __init__(self, name:str=None, config_filename=None, RUN_IN_THREAD=True):
54 52 if name is None: name = self.__class__.__name__
55   - super().__init__(name, config_filename, self.RUN_IN_THREAD)
  53 + super().__init__(name, config_filename, RUN_IN_THREAD)
56 54  
57 55 # @override
58 56 def init(self):
... ... @@ -159,3 +157,23 @@ class AgentA(Agent):
159 157 def exec_specific_cmd_end(self, cmd:Command, from_thread=True):
160 158 super().exec_specific_cmd_end(cmd, from_thread)
161 159 '''
  160 +
  161 +
  162 +
  163 +if __name__ == "__main__":
  164 +
  165 + # with thread
  166 + RUN_IN_THREAD=True
  167 + # with process
  168 + RUN_IN_THREAD=False
  169 +
  170 + configfile = None
  171 +
  172 + # arg 1 : config file
  173 + if len(sys.argv) == 2:
  174 + configfile = sys.argv[1]
  175 +
  176 + #agent = AgentX()
  177 + agent = AgentA("AgentA", configfile, RUN_IN_THREAD)
  178 + print(agent)
  179 + agent.run()
... ...
src/agent/AgentX.py 100644 → 100755
  Diff comments   View file @a788702
  1 +#!/usr/bin/env python3
1 2  
2   -import utils.Logger as L
3   -import threading #, multiprocessing, os
4   -import time
  3 +import sys
  4 +##import utils.Logger as L
  5 +#import threading #, multiprocessing, os
  6 +#import time
5 7  
6   -from django.db import transaction
7   -from common.models import Command
  8 +#from django.db import transaction
  9 +#from common.models import Command
8 10  
9   -from .Agent import Agent
  11 +##from .Agent import Agent
  12 +from Agent import Agent
10 13  
11 14  
12 15  
13   -log = L.setupLogger("AgentXTaskLogger", "AgentX")
  16 +##log = L.setupLogger("AgentXTaskLogger", "AgentX")
14 17  
15 18  
16 19  
17 20 class AgentX(Agent):
18 21  
19 22 """
20   - How to run this agent thread_exec_specific_cmd() method ?
21   - - True = inside a Thread (cannot be killed, must be asked to stop, and inadequate for computation)
22   - - False = inside a Process
23   - If thread, displays :
24   - >>>>> Thread: starting execution of command specific1
25   - >>>>> Thread: PID: 2695, Process Name: MainProcess, Thread Name: Thread-1
26   - ...
27   - >>>>> Thread: starting execution of command specific2
28   - >>>>> Thread: PID: 2695, Process Name: MainProcess, Thread Name: Thread-2
29   - ...
30   - >>>>> Thread: starting execution of command specific3
31   - >>>>> Thread: PID: 2695, Process Name: MainProcess, Thread Name: Thread-3
32   - If process, displays :
33   - >>>>> Thread: starting execution of command specific1
34   - >>>>> Thread: PID: 2687, Process Name: Process-1, Thread Name: MainThread
35   - ...
36   - >>>>> Thread: starting execution of command specific2
37   - >>>>> Thread: PID: 2689, Process Name: Process-2, Thread Name: MainThread
38   - ...
39   - >>>>> Thread: starting execution of command specific3
40   - >>>>> Thread: PID: 2690, Process Name: Process-3, Thread Name: MainThread
41   - """
42   - RUN_IN_THREAD = True
43   - RUN_IN_THREAD = False
44   -
45   -
46   - """
47 23 =================================================================
48 24 FUNCTIONS RUN INSIDE MAIN THREAD
49 25 =================================================================
50 26 """
51 27  
52 28 # @override
53   - def __init__(self, name:str=None, config_filename=None):
  29 + def __init__(self, name:str=None, config_filename=None, RUN_IN_THREAD=True):
54 30 if name is None: name = self.__class__.__name__
55   - super().__init__(name, config_filename, self.RUN_IN_THREAD)
  31 + super().__init__(name, config_filename, RUN_IN_THREAD)
56 32  
57 33 # @override
58 34 def init(self):
... ... @@ -159,3 +135,22 @@ class AgentX(Agent):
159 135 def exec_specific_cmd_end(self, cmd:Command, from_thread=True):
160 136 super().exec_specific_cmd_end(cmd, from_thread)
161 137 '''
  138 +
  139 +
  140 +if __name__ == "__main__":
  141 +
  142 + # with thread
  143 + RUN_IN_THREAD=True
  144 + # with process
  145 + RUN_IN_THREAD=False
  146 +
  147 + configfile = None
  148 +
  149 + # arg 1 : config file
  150 + if len(sys.argv) == 2:
  151 + configfile = sys.argv[1]
  152 +
  153 + #agent = AgentX()
  154 + agent = AgentX("AgentX", configfile, RUN_IN_THREAD)
  155 + print(agent)
  156 + agent.run()
... ...
start_agent.py
  Diff comments   View file @a788702
... ... @@ -100,6 +100,12 @@ if agent_name == &quot;agentA&quot;:
100 100 agentA.run(FOR_REAL=True)
101 101 sys.exit(0)
102 102  
  103 +"""
  104 +from src.agent.Agent import Agent
  105 +agent = Agent(name="agent", config_filename=configfile)
  106 +agent.run(FOR_REAL=True)
  107 +"""
  108 +
103 109 # Default agent is AgentX
104 110  
105 111 from src.agent.AgentX import AgentX
... ...