Fixing AgentCmd create (create_cmd_for), Fixing init of device_command to parse …

…cmd full_name into cmd name and args, Fixing agentSST methods / commands

Fixing AgentCmd create (create_cmd_for), Fixing init of device_command to parse …
…cmd full_name into cmd name and args, Fixing agentSST methods / commands
Alexis Koralewski
1 parent 82b3357a
Showing 4 changed files with 79 additions and 50 deletions Show diff stats
CHANGELOG
src/core/pyros_django/agent/Agent.py
src/core/pyros_django/agent/AgentSST.py
src/device_controller/abstract_component/device_controller.py
 25-05-2022 (AKo): v0.4.10.0
 	- Updating Agent constructor (removing RUNNIN_IN_THREAD)
 	- Adding new start for pyros.py (launching agentSST)
+	- Add option foreground for new-start (if set, doesn't show output of pyros start)
+	- Fixing AgentCmd create (create_cmd_for), Fixing init of device_command to parse cmd full_name into cmd name and args, Fixing agentSST methods/commands
  
 18-05-2022 (AKo): v0.4.10.0
 	- Adding mandatory agents in obsconfig
@@ -936,7 +936,7 @@ class Agent:
         if cmd:
             log.info('-'*6)
             log.info('-'*6 + " RECEIVED NEW COMMAND TO PROCESS: ")
-            log.info('-'*6 + cmd)
+            log.info('-'*6 + str(cmd))
             log.info('-'*6)            
  
             # CASE 1 - AGENT LEVEL command
@@ -1325,13 +1325,13 @@ class Agent:
             real_agent_name = self._get_real_agent_name(to_agent)
         except KeyError as e:
         '''
-        real_agent_name = self._get_real_agent_name(to_agent)
-        if not real_agent_name:
-            log.e("UNKNOWN AgentDevice ALIAS", to_agent)
-            #self.log_e("Exception raised", e)
-            log.e(f"=> Thus, I do not send this command '{cmd_name}'")
-            return None
-        return AgentCmd.create(self.name, real_agent_name, cmd_name, cmd_args)
+        # real_agent_name = self._get_real_agent_name(to_agent)
+        # if not real_agent_name:
+        #     log.e("UNKNOWN AgentDevice ALIAS", to_agent)
+        #     #self.log_e("Exception raised", e)
+        #     log.e(f"=> Thus, I do not send this command '{cmd_name}'")
+        #     return None
+        return AgentCmd.create(self.name, to_agent, cmd_name, cmd_args)
         '''
         return AgentCmd(
             sender=self.name,
@@ -2,7 +2,8 @@
  
 from pathlib import Path
 import subprocess
-import sys, os, datetime
+import sys, os
+from datetime import datetime, timezone, timedelta
 ##import utils.Logger as L
 #import threading #, multiprocessing, os
 import time
@@ -12,7 +13,7 @@ import time
  
 from Agent import Agent, build_agent, log, extract_parameters
 import socket
-from common.models import AgentCmd
+from common.models import AgentCmd, AgentSurvey
  
 from src.core.pyros_django.obsconfig.obsconfig_class import OBSConfig
  
@@ -24,8 +25,8 @@ class AgentSST(Agent):
     subprocess_dict = {}
  
     AGENT_SPECIFIC_COMMANDS = [
-        "KILL_AGENT",
-        "RESTART_AGENT"
+        "kill_agent",
+        "restart_agent"
     ]
  
     def __init__(self, name:str=None,sim_computer=None):
@@ -101,7 +102,8 @@ class AgentSST(Agent):
                     if os.path.exists(protocol_folder_abs_path + os.sep + protocol_script_name):
                         cmd = self.VENV_PYTHON +" "+ protocol_folder_abs_path + os.sep + protocol_script_name
  
-                        process = subprocess.Popen(f"{cmd}", shell=True, stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT)
+                        # process = subprocess.Popen(f"{cmd}", shell=True, stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT)
+                        process = subprocess.Popen(f"{cmd}", shell=True)
                         self.subprocess_dict[agent] = process
                         log.info(f"Start agent {agent} with cmd {cmd}")
  
@@ -115,54 +117,78 @@ class AgentSST(Agent):
         self.start_agents(agent_name)
  
  
-    def cmd_kill_agent(self, cmd:AgentCmd, agent:str)->None:
+    def kill_agent(self, *args)->None:
+        agent = args[0]
         if agent in self.subprocess_dict.keys():
-            self.send_cmd_to(agent,"do_exit")
+            cmd = self.send_cmd_to(agent,"do_exit")
+            return cmd
  
-    def cmd_restart_agent(self, cmd:AgentCmd ,agent:str)->None:
+    def restart_agent(self, *args)->None:
+        agent = args[0]
+        self.kill_agent(agent)
         self.start_agent(agent)
-        if agent in self.subprocess_dict.keys():
-            cmd.set_result(f"Agent {agent} restarted")
-            cmd.set_as_processed()
-        else:
-            cmd.set_result(f"Agent {agent} failed to restart")
-            log.debug(f"Agent {agent} failed to restart")
+        # if agent in self.subprocess_dict.keys():
+        #     cmd.set_result(f"Agent {agent} restarted")
+        #     cmd.set_as_processed()
+        # else:
+        #     cmd.set_result(f"Agent {agent} failed to restart")
+        #     log.debug(f"Agent {agent} failed to restart")
  
-    def force_kill_agent(self, cmd:AgentCmd, agent:str)->None:
+    def force_kill_agent(self, *args)->None:
         if agent in self.subprocess_dict.get(agent) is not None:
             process = self.subprocess_dict.get(agent)
             process.terminate()
+            process.wait()
             if process.poll() is not None:
-                cmd.set_as_processed()
+                pass
+                #cmd.set_as_processed()
         else:
             return None
-            
+
+    def do_things_before_exit(self,abort_cmd_sender):
+        kill_agent_commands = {}
+        for agent in self.subprocess_dict.keys():
+            cmd = self.kill_agent(agent)    
+            kill_agent_commands[agent] = cmd
+        for agent in kill_agent_commands:
+            if self.subprocess_dict[agent].poll() is None:
+                self.force_kill_agent(kill_agent_commands[agent],agent)
+
     def routine_process_before_body(self):
-        try:
-            now_time = datetime.datetime.utcnow()
-            last_running_commands = AgentCmd.get_commands_sent_by_agent("AgentSST").filter(state="CMD_RUNNING")
-
-            for cmd in last_running_commands:
-                last_running_cmd, agent = cmd.full_name.split(" ")[:1]
-                if last_running_cmd == "KILL_AGENT" and cmd.is_expired():
-                    self.force_killed_agent(agent)
-            next_cmd = self._get_next_valid_and_not_running_command()
-            if next_cmd is not None:
-                cmd, agent = next_cmd.full_name.split(" ")[:1]
-                if cmd == "KILL_AGENT":
-                    self.cmd_kill_agent(next_cmd,agent)
-                elif cmd == "RESTART_AGENT":
-                    self.cmd_restart_agent(next_cmd,agent)
-            log.info("Check status of process")
-            for agent in self.subprocess_dict:
-                proc = self.subprocess_dict.get(agent)
+        now_time = datetime.now(timezone.utc) 
+        last_running_commands = AgentCmd.get_commands_sent_by_agent("AgentSST").filter(state="CMD_RUNNING")
+
+        for cmd in last_running_commands:
+            last_running_cmd, agent = cmd.full_name.split(" ")[:1]
+            if last_running_cmd == "KILL_AGENT" and cmd.is_expired():
+                self.force_killed_agent(agent)
+        # chekcing status of agent if they are timeout
+
+        for agent in self.subprocess_dict.keys():
+            agent_survey = AgentSurvey.objects.get(name=agent)
+            validity_duration = agent_survey.validity_duration
+            last_update_from_agent = agent_survey.updated
+            validity_duration_timedelta = timedelta(seconds=validity_duration)
+            timeout_datetime = last_update_from_agent + validity_duration_timedelta
+            timeout_datetime = timeout_datetime.replace(tzinfo=timezone.utc)
+            # if agent is timeout, restart it
+            if timeout_datetime < now_time:
+                # restart agent 
+                self.send_cmd_to("AgentSST","restart_agent", agent)
+                
+                
+        # next_cmd = self._get_next_valid_and_not_running_command()
+        # if next_cmd is not None:
+        #     cmd, agent = next_cmd.full_name.split(" ")[:1]
+        #     if cmd == "KILL_AGENT":
+        #         self.cmd_kill_agent(next_cmd,agent)
+        #     elif cmd == "RESTART_AGENT":
+        #         self.cmd_restart_agent(next_cmd,agent)
+        log.info("Check status of process")
+        for agent in self.subprocess_dict:
+            proc = self.subprocess_dict.get(agent)
             log.info(f"{agent} poll result is {proc.poll()}")
-        except KeyboardInterrupt:
-
-            for process in self.subprocess_dict.values():
-                print(process)
-                process.kill()
-            exit(0)
+     
  
  
 if __name__ == "__main__":
@@ -113,7 +113,8 @@ class DeviceCmd:
  
     def __init__(self, cmd_full_name:str, dev_comp_type:str=None, cmd_args:str=None):
         self.full_name = cmd_full_name
-        self.name = cmd_full_name
+        dev_comp_type,cmd_name,cmd_args = self.get_full_name_parts()
+        self.name = cmd_name
         self.devtype = dev_comp_type
         self.args = cmd_args
         if self.is_generic():