Blame view

src/core/pyros_django/agent/AgentSST.py 16.4 KB
05316241   Alexis Koralewski   Adding AgentSST, ...
1
2
3
4
#!/usr/bin/env python3

from pathlib import Path
import subprocess
9cd82394   Alexis Koralewski   Add new option fo...
5
6
import sys, os, argparse

a04e004d   Alexis Koralewski   Fixing AgentCmd c...
7
from datetime import datetime, timezone, timedelta
05316241   Alexis Koralewski   Adding AgentSST, ...
8
9
10
11
12
13
14
##import utils.Logger as L
#import threading #, multiprocessing, os
import time

#from django.db import transaction
#from common.models import Command

dd27c2bc   Alexis Koralewski   Updating agent co...
15
from Agent import Agent, build_agent, log, extract_parameters
05316241   Alexis Koralewski   Adding AgentSST, ...
16
import socket
c86245ec   Alexis Koralewski   AgentSST not long...
17
from common.models import AgentCmd, AgentSurvey, Majordome
05316241   Alexis Koralewski   Adding AgentSST, ...
18

dd27c2bc   Alexis Koralewski   Updating agent co...
19
from src.core.pyros_django.obsconfig.obsconfig_class import OBSConfig
05316241   Alexis Koralewski   Adding AgentSST, ...
20
21

class AgentSST(Agent):
04b0b442   Alexis Koralewski   Changing computer...
22
    computer = "XCY1"
05316241   Alexis Koralewski   Adding AgentSST, ...
23
24
25
26
    _previous_dir = ""
    PROJECT_ROOT_PATH = ""
    VENV_PYTHON = ""
    subprocess_dict = {}
7fd15ce5   Alexis Koralewski   Adding test mode ...
27
    agent_in_mode_test = {}
05316241   Alexis Koralewski   Adding AgentSST, ...
28

dd27c2bc   Alexis Koralewski   Updating agent co...
29
    AGENT_SPECIFIC_COMMANDS = [
1de7129b   Alexis Koralewski   Adapt agentSST to...
30
31
32
        ("do_kill_agent",10,0),
        ("do_restart_agent",20,0),
        ("do_start_agent",20,0),
dd27c2bc   Alexis Koralewski   Updating agent co...
33
    ]
7fd15ce5   Alexis Koralewski   Adding test mode ...
34
    TEST_COMMANDS_LIST = [
c86245ec   Alexis Koralewski   AgentSST not long...
35
36
        ("self get_mode", 200, "MODE = ATTENTIVE",  Agent.CMD_STATUS.CMD_EXECUTED),
        ("self set_mode ATTENTIVE", 200, "MODE = ATTENTIVE", Agent.CMD_STATUS.CMD_EXECUTED),
7fd15ce5   Alexis Koralewski   Adding test mode ...
37
    ]
05316241   Alexis Koralewski   Adding AgentSST, ...
38

d6a94dfd   Alexis Koralewski   Updating AgentSST...
39
    def __init__(self, name:str=None, simulated_computer=None, agent=None):
05316241   Alexis Koralewski   Adding AgentSST, ...
40
        
dd27c2bc   Alexis Koralewski   Updating agent co...
41
        super().__init__()
05316241   Alexis Koralewski   Adding AgentSST, ...
42
43
44
        self.PROJECT_ROOT_PATH = os.environ["PROJECT_ROOT_PATH"]
        if name is None:
            name = self.__class__.__name__
6fa0909e   Alexis Koralewski   Add button to man...
45
            self.name = name
4a81e038   Alexis Koralewski   Add get_specifics...
46
        self.computer = socket.gethostname()    
d6a94dfd   Alexis Koralewski   Updating AgentSST...
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
        if simulated_computer != None:
            self.computer = simulated_computer
        name_from_config = self.get_config().get_agent_sst_of_computer(self.computer)
        if name_from_config != None:
            name = name_from_config
            self.name = name
        if AgentSurvey.objects.filter(name=self.name).exists():
            self._agent_survey = AgentSurvey.objects.get(name=self.name)
        else:
            self._agent_survey = AgentSurvey.objects.create(
                name=self.name, 
                validity_duration=60, 
                mode=self.mode, 
                status=self.status, 
                iteration=-1
            )
05316241   Alexis Koralewski   Adding AgentSST, ...
63
64
65
        WITH_DOCKER = False
        if os.environ.get("WITH_DOCKER"):
            WITH_DOCKER = True
99502363   Alexis Koralewski   add message when ...
66
            # if WITH_DOCKER socket.gethostname() bizarre 
05316241   Alexis Koralewski   Adding AgentSST, ...
67
68
69
70
71
72
73
74
75
76
77
78
79
80
        if WITH_DOCKER:
            VENV_ROOT = ""
            VENV = ""
            VENV_BIN = ""
        else:
            VENV_ROOT = "venv"
            VENV = "venv_py3_pyros"
            VENV_BIN = (
                self.PROJECT_ROOT_PATH
                + os.sep + VENV_ROOT
                + os.sep + VENV
                + os.sep + "bin"
                + os.sep
            )
dd27c2bc   Alexis Koralewski   Updating agent co...
81
        self.VENV_PYTHON = VENV_BIN + "python3"
9cd82394   Alexis Koralewski   Add new option fo...
82
83
84
85
        if agent:
            self.init_agent = agent
        else:
            self.init_agent = None
c86245ec   Alexis Koralewski   AgentSST not long...
86
        self._no_restart = False
4a81e038   Alexis Koralewski   Add get_specifics...
87
        #log.info(f"PC hostname is {self.computer}")
7fd15ce5   Alexis Koralewski   Adding test mode ...
88
89
90
    
    def init(self):
        super().init()
4a81e038   Alexis Koralewski   Add get_specifics...
91
        log.info(f"PC hostname is {self.computer}")
9cd82394   Alexis Koralewski   Add new option fo...
92
93
94
95
        if self.init_agent is None:
            self.start_agents()
        else:
            self.start_agents(self.init_agent)
c86245ec   Alexis Koralewski   AgentSST not long...
96
97
        if self.TEST_MODE:
            self._no_restart = True
7fd15ce5   Alexis Koralewski   Adding test mode ...
98
        self.TEST_MODE = False
ea1ca112   Alexis Koralewski   Fixing agentSST s...
99
100
        time.sleep(10)
        self.set_delay(3)
a3d0b4b0   Alexis Koralewski   AgentSST : adapti...
101

4a81e038   Alexis Koralewski   Add get_specifics...
102
103
    def set_computer(self,computer):
        self.computer = computer
05316241   Alexis Koralewski   Adding AgentSST, ...
104
105
106
107
108
109
110
111
    def start_agents(self,agent_name=None):
        """
        Start all agents or one agent of obs_config

        Args:
            agent_name (_type_, optional): Specific agent name to start. Defaults to None.
        """
        obs_config = self.get_config()
7fd15ce5   Alexis Koralewski   Adding test mode ...
112
113
        test_mode = " -t"

05316241   Alexis Koralewski   Adding AgentSST, ...
114
115
        if agent_name:
            agent = agent_name
9d8855ba   Alexis Koralewski   Add three agentss...
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
            if "AgentSST" not in agent:
                agents = obs_config.get_agents_per_computer(obs_config.unit_name).get(self.computer)
                if agent not in agents:
                    log.info(f"{agent} isn't associated to this computer : {self.computer}")
                    log.info(f"Agents associated to this computer : {agents}")
                    exit(1)
                # Start a specific agent of obs_config (restart)
                agent_informations = obs_config.get_agent_information(obs_config.unit_name,agent)
                protocol = agent_informations.get("protocol")
                if protocol:
                    protocol_folder_abs_path = os.path.join(self.PROJECT_ROOT_PATH, os.path.dirname(protocol))
                    
                    protocol_script_name = protocol.split("/")[-1]
                    if os.path.exists(protocol_folder_abs_path + os.sep + protocol_script_name):
                        cmd = self.VENV_PYTHON +" "+ protocol_folder_abs_path + os.sep + protocol_script_name
                        if not agent in self.agent_in_mode_test:
                            self.agent_in_mode_test[agent] = self.TEST_MODE
                        if self.agent_in_mode_test[agent]:
                            cmd += test_mode
                        process = subprocess.Popen(f"{cmd}",shell=True)
                        process.poll()
                        if agent not in self.subprocess_dict:
                            self.subprocess_dict[agent] = {}
                        self.subprocess_dict[agent]["process"] = process
                        nb_try_restart = self.subprocess_dict[agent].get("nb_try_restart",0)
                        nb_try_restart += 1
                        self.subprocess_dict[agent]["nb_try_restart"] = nb_try_restart
                        log.info(f"Start agent {agent} with cmd {cmd}")
05316241   Alexis Koralewski   Adding AgentSST, ...
144
145

        else:
027eaa78   Alexis Koralewski   Fixing AgentSST i...
146
147
148
149
150
151
152
153
154
            agents = obs_config.get_agents_per_computer(obs_config.unit_name).get(self.computer)
            if agents is None:
                available_hostnames = obs_config.get_agents_per_computer(obs_config.unit_name).keys()
                log.info("Computer not found in obs config")
                log.info(f"Available hostnames {available_hostnames}. Current hostname is {self.computer}")
                exit(1)
            #self.change_dir(self.PROJECT_ROOT_PATH)
            else:
                log.info(f"Agents associated to this computer : {agents}")
05316241   Alexis Koralewski   Adding AgentSST, ...
155
156
            # Start every agent of obs_config (initial start)
            for agent in agents:
9d8855ba   Alexis Koralewski   Add three agentss...
157
158
                if "AgentSST" in agent:
                    continue
05316241   Alexis Koralewski   Adding AgentSST, ...
159
160
161
162
163
164
165
                agent_informations = obs_config.get_agent_information(obs_config.unit_name,agent)
                protocol = agent_informations.get("protocol")
                if protocol:
                    protocol_folder_abs_path = os.path.join(self.PROJECT_ROOT_PATH, os.path.dirname(protocol))
                    
                    protocol_script_name = protocol.split("/")[-1]
                    if os.path.exists(protocol_folder_abs_path + os.sep + protocol_script_name):
05316241   Alexis Koralewski   Adding AgentSST, ...
166
                        
7fd15ce5   Alexis Koralewski   Adding test mode ...
167
168
169
170
171
                        cmd = self.VENV_PYTHON +" "+ protocol_folder_abs_path + os.sep + protocol_script_name
                        if not agent in self.agent_in_mode_test:
                            self.agent_in_mode_test[agent] = self.TEST_MODE
                        if self.agent_in_mode_test[agent]:
                            cmd += test_mode
a04e004d   Alexis Koralewski   Fixing AgentCmd c...
172
173
                        # process = subprocess.Popen(f"{cmd}", shell=True, stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT)
                        process = subprocess.Popen(f"{cmd}", shell=True)
9bd7ac9e   Alexis Koralewski   Adding timeout co...
174
175
                        self.subprocess_dict[agent] = {}
                        self.subprocess_dict[agent]["process"] = process
3449489e   Alexis Koralewski   Fixing current_nb...
176
177
                        # Reset to zero nb_try when AgentSST start (launch all agents)
                        self.subprocess_dict[agent]["nb_try_restart"] = 0
05316241   Alexis Koralewski   Adding AgentSST, ...
178
179
                        log.info(f"Start agent {agent} with cmd {cmd}")

2918ab51   Alexis Koralewski   Adding do_start_a...
180
    def do_start_agent(self, agent_name:str):
05316241   Alexis Koralewski   Adding AgentSST, ...
181
182
183
184
185
186
187
        """
        Start a specific agent of obs_config (Restart)

        Args:
            agent_name (str): Name of agent to start
        """
        self.start_agents(agent_name)
3449489e   Alexis Koralewski   Fixing current_nb...
188
189
190
191
192
        nb_try_restart_agent = self.subprocess_dict[agent_name]["nb_try_restart"]
        agent_survey = AgentSurvey.objects.get(name=agent_name)
        agent_survey.current_nb_restart = nb_try_restart_agent
        
        agent_survey.save()
05316241   Alexis Koralewski   Adding AgentSST, ...
193

dd27c2bc   Alexis Koralewski   Updating agent co...
194

1de7129b   Alexis Koralewski   Adapt agentSST to...
195
    def do_kill_agent(self, agent:str)->None:
afbc5c95   Alexis Koralewski   Renaming AgentSST...
196
        # agent = args[0]
6fa0909e   Alexis Koralewski   Add button to man...
197
        if agent in self.subprocess_dict.keys() or agent == self.name:
9fa0958e   Alexis Koralewski   Adding do_restart...
198
199
            #cmd = self.send_cmd_to(agent,"do_exit")
            cmd = self.send_cmd_to(agent,"do_stop","asap")
a04e004d   Alexis Koralewski   Fixing AgentCmd c...
200
            return cmd
dd27c2bc   Alexis Koralewski   Updating agent co...
201

9fa0958e   Alexis Koralewski   Adding do_restart...
202
    def do_restart_agent(self, agent:str, mode:str)->None:
afbc5c95   Alexis Koralewski   Renaming AgentSST...
203
        # agent = args[0]
9bd7ac9e   Alexis Koralewski   Adding timeout co...
204
        nb_try_restart_agent = self.subprocess_dict[agent]["nb_try_restart"]
c957b9e1   Alexis Koralewski   fixing restart of...
205
        if nb_try_restart_agent < AgentSurvey.objects.get(name=agent).nb_restart_max:
9fa0958e   Alexis Koralewski   Adding do_restart...
206
207
208
209
210
            if mode == "soft":
                cmd = self.send_cmd_to(agent,"do_restart",mode)
            else:
                self.do_kill_agent(agent)
                self.do_start_agent(agent)
9bd7ac9e   Alexis Koralewski   Adding timeout co...
211
212
213
        else:
            #sendmail
            pass
a3d0b4b0   Alexis Koralewski   AgentSST : adapti...
214
215
216
        agent_survey = AgentSurvey.objects.get(name=agent)
        agent_survey.current_nb_restart = nb_try_restart_agent
        agent_survey.save()
9bd7ac9e   Alexis Koralewski   Adding timeout co...
217

a04e004d   Alexis Koralewski   Fixing AgentCmd c...
218
219
220
221
222
223
        # if agent in self.subprocess_dict.keys():
        #     cmd.set_result(f"Agent {agent} restarted")
        #     cmd.set_as_processed()
        # else:
        #     cmd.set_result(f"Agent {agent} failed to restart")
        #     log.debug(f"Agent {agent} failed to restart")
dd27c2bc   Alexis Koralewski   Updating agent co...
224

a04e004d   Alexis Koralewski   Fixing AgentCmd c...
225
    def force_kill_agent(self, *args)->None:
d4ebe565   Alexis Koralewski   Adding pyros stop...
226
227
228
        if args:
            agent = args[0]
            if self.subprocess_dict.get(agent) is not None:
9bd7ac9e   Alexis Koralewski   Adding timeout co...
229
                process = self.subprocess_dict.get(agent).get("process")
d4ebe565   Alexis Koralewski   Adding pyros stop...
230
231
232
233
234
235
                # process.terminate()
                # process.wait()
                # Kill is better when using Popen(shell=True) because it will remove the created child process
                process.kill()
            else:
                return None
a04e004d   Alexis Koralewski   Fixing AgentCmd c...
236
237
238
239

    def do_things_before_exit(self,abort_cmd_sender):
        kill_agent_commands = {}
        for agent in self.subprocess_dict.keys():
8d9c8345   Alexis Koralewski   Renaming commands...
240
            cmd = self.do_kill_agent(agent)    
a04e004d   Alexis Koralewski   Fixing AgentCmd c...
241
            kill_agent_commands[agent] = cmd
3449489e   Alexis Koralewski   Fixing current_nb...
242
243
244
            agent_survey = AgentSurvey.objects.get(name=agent)
            # Reset counter before exiting
            agent_survey.current_nb_restart = 0
a3d0b4b0   Alexis Koralewski   AgentSST : adapti...
245
            agent_survey.save()                             
d4ebe565   Alexis Koralewski   Adding pyros stop...
246
        # wait 10 seconds in order to agents to exit themselves properly 
47a1e5b7   Alexis Koralewski   Fixing restarting...
247
        time.sleep(20)
ea1ca112   Alexis Koralewski   Fixing agentSST s...
248
249
250
        for agent in self.subprocess_dict.keys():
            while self.subprocess_dict[agent].get("process").poll() is None:
                time.sleep(0.5)
65c091a2   Alexis Koralewski   Update variable n...
251
252
253
        # agent_survey = AgentSurvey.objects.get(name=self.name)
        # agent_survey.status = AgentSurvey.STATUS_EXIT
        # agent_survey.save()
2ac0a02f   Alexis Koralewski   Fixing AgentSST (...
254
    def routine_process_after_body(self):
a04e004d   Alexis Koralewski   Fixing AgentCmd c...
255
        now_time = datetime.now(timezone.utc) 
027eaa78   Alexis Koralewski   Fixing AgentSST i...
256
        last_running_commands = AgentCmd.get_commands_sent_by_agent("AgentSST").filter(state="CMD_RUNNING",recipient__in=list(self.subprocess_dict.keys()))
a04e004d   Alexis Koralewski   Fixing AgentCmd c...
257
        for cmd in last_running_commands:
fd880010   Alexis Koralewski   Fixing error in A...
258
            last_running_cmd = cmd.full_name
a04e004d   Alexis Koralewski   Fixing AgentCmd c...
259
            if last_running_cmd == "KILL_AGENT" and cmd.is_expired():
fd880010   Alexis Koralewski   Fixing error in A...
260
                agent = cmd.args[0]
2918ab51   Alexis Koralewski   Adding do_start_a...
261
                self.force_kill_agent(agent)
a04e004d   Alexis Koralewski   Fixing AgentCmd c...
262

9cd82394   Alexis Koralewski   Add new option fo...
263
        # checking status of agent if they are timeout if in auto mode
c86245ec   Alexis Koralewski   AgentSST not long...
264
265
266
267
        try:
            soft_mode = Majordome.objects.last().soft_mode
        except Majordome.DoesNotExist:
            soft_mode = None
47a1e5b7   Alexis Koralewski   Fixing restarting...
268
        if (soft_mode is not None and soft_mode == "AUTO") and not self._no_restart:
c86245ec   Alexis Koralewski   AgentSST not long...
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
            for agent in self.subprocess_dict.keys():
                try:
                    agent_survey = AgentSurvey.objects.get(name=agent)
                except AgentSurvey.DoesNotExist:
                    # If there is no entry in AgentSurvey for this agent go to next iteration (it surely means that the agentSST launched this agent for the first time, and it didn't had enough time to create an entry in AgentSurvey)
                    continue
                
                validity_duration = agent_survey.validity_duration
                last_update_from_agent = agent_survey.updated
                validity_duration_timedelta = timedelta(seconds=validity_duration)
                timeout_datetime = last_update_from_agent + validity_duration_timedelta
                timeout_datetime = timeout_datetime.replace(tzinfo=timezone.utc)
                # if agent latest state is timeout, restart it
                if timeout_datetime < now_time:
                    if self.subprocess_dict[agent].get("process").poll() != None:
                        last_executed_start_agent_cmd =  AgentCmd.objects.filter(state="CMD_EXECUTED",full_name=f"do_start_agent {agent}",recipient=self.name).order_by("-s_deposit_time")
                        if last_executed_start_agent_cmd.exists():
                            cmd_outdated_datetime_start = datetime.utcnow() - timedelta(seconds=30)
                            cmd_outdated_datetime_end = datetime.utcnow() - timedelta(seconds=25)
                            cmd_outdated_datetime_start = cmd_outdated_datetime_start.replace(tzinfo=timezone.utc)
                            cmd_outdated_datetime_end = cmd_outdated_datetime_end.replace(tzinfo=timezone.utc)
                            # cmd outdated if deposit time was between 25 and 30 seconds ago from now
                            # if last start cmd for this agent was executed and this agent isn't currently running, ask again a start.
                            if cmd_outdated_datetime_start >= last_executed_start_agent_cmd.first().s_deposit_time and cmd_outdated_datetime_end >= last_executed_start_agent_cmd.first().s_deposit_time:
652d479b   Alexis Koralewski   Adding titles, c...
293
                                self.send_cmd_to(self.name,"do_start_agent", agent)
c86245ec   Alexis Koralewski   AgentSST not long...
294
295
296
297
298
299
                        else:
                            try:
                                # Check if do_start_agent cmd already asked by agentSST in previous iterations and not exectuted. If the query success (no exception raised), we don't send again a cmd
                                AgentCmd.get_pending_and_running_commands_for_agent(self.name).get(full_name=f"do_start_agent {agent}")
                            except:
                                self.send_cmd_to(self.name,"do_start_agent", agent)
027eaa78   Alexis Koralewski   Fixing AgentSST i...
300
                    else:
c86245ec   Alexis Koralewski   AgentSST not long...
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
                        last_executed_start_or_restart_agent_cmd =  AgentCmd.objects.filter(state="CMD_EXECUTED",full_name__in=(f"do_start_agent {agent}",f"do_restart_agent {agent}"),recipient=self.name).order_by("-s_deposit_time")
                        if last_executed_start_or_restart_agent_cmd.exists():
                            cmd_outdated_datetime_start = datetime.utcnow() - timedelta(seconds=30)
                            cmd_outdated_datetime_end = datetime.utcnow() - timedelta(seconds=25)
                            cmd_outdated_datetime_start = cmd_outdated_datetime_start.replace(tzinfo=timezone.utc)
                            cmd_outdated_datetime_end = cmd_outdated_datetime_end.replace(tzinfo=timezone.utc)
                            # cmd outdated if deposit time was between 25 and 30 seconds ago from now
                            # if last start or restart cmd for this agent was executed and this agent isn't currently running, ask again a restart.
                            if cmd_outdated_datetime_start >= last_executed_start_or_restart_agent_cmd.first().s_deposit_time and cmd_outdated_datetime_end >=  last_executed_start_or_restart_agent_cmd.first().s_deposit_time:
                                self.send_cmd_to(self.name,"do_restart_agent", agent)    
                        else:
                            try:
                                # Check if do_restart_agent cmd already asked by agentSST in previous iterations and not exectuted. If the query success (no exception raised), we don't send again a cmd
                                AgentCmd.get_pending_and_running_commands_for_agent(self.name).get(full_name=f"do_restart_agent {agent}")
                            except:
                                self.send_cmd_to(self.name,"do_restart_agent", agent)
027eaa78   Alexis Koralewski   Fixing AgentSST i...
317

a04e004d   Alexis Koralewski   Fixing AgentCmd c...
318
319
        log.info("Check status of process")
        for agent in self.subprocess_dict:
9bd7ac9e   Alexis Koralewski   Adding timeout co...
320
            proc = self.subprocess_dict.get(agent).get("process")
dd27c2bc   Alexis Koralewski   Updating agent co...
321
            log.info(f"{agent} poll result is {proc.poll()}")
a04e004d   Alexis Koralewski   Fixing AgentCmd c...
322
     
dd27c2bc   Alexis Koralewski   Updating agent co...
323

05316241   Alexis Koralewski   Adding AgentSST, ...
324
325

if __name__ == "__main__":
9cd82394   Alexis Koralewski   Add new option fo...
326
327
328
329
330
331
332
    parser = argparse.ArgumentParser(description='Start a agentSST.')
    parser.add_argument("--computer",dest="computer",help='Launch agent with simulated computer hostname',action="store")
    parser.add_argument("--agent",dest="agent",help='Launch an specific agent ',action="store")
    parser.add_argument("-t", action="store_true" )
    args = vars(parser.parse_args())
    agent = build_agent(AgentSST,param_constr=args)
    # agent = build_agent(AgentSST)
05316241   Alexis Koralewski   Adding AgentSST, ...
333
    agent.run()