From 03814884c0f73a0e4fcc8dfe097b04d1af4835c8 Mon Sep 17 00:00:00 2001 From: Etienne Pallier Date: Fri, 20 Jul 2018 15:41:22 +0200 Subject: [PATCH] launch majordome agent from majordome unit tests --- pyros.py | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- src/majordome/start_agent_majordome.py | 5 ++++- src/majordome/start_agent_majordome_from_test.py | 26 ++++++++++++++++++++++++++ src/majordome/tasks.py | 29 +++++++++++++++++++---------- src/majordome/tests.py | 104 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----- src/misc/fixtures/initial_fixture.json | 3 +++ src/misc/fixtures/tests/majordome_test.json | 50 ++++++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 279 insertions(+), 19 deletions(-) create mode 100755 src/majordome/start_agent_majordome_from_test.py create mode 100644 src/misc/fixtures/tests/majordome_test.json diff --git a/pyros.py b/pyros.py index c76b62c..c442322 100755 --- a/pyros.py +++ b/pyros.py @@ -11,8 +11,8 @@ import signal # For USE_CELERY global variable #from src.pyros import settings -USE_CELERY = False -#USE_CELERY = True +#USE_CELERY = False +USE_CELERY = True DEBUG = True SIMULATOR_CONFIG_FILE = "conf.json" #default config file @@ -679,6 +679,7 @@ class Pyros(AManager): # EP added: if not TOTAL: self.reset_database_sim() # Actualise l’état de la DB en accord avec l’ensemble des modèles et des migrations actuels + self.makemigrations() self.migrate() # Load fixture initial_fixture.json self.loaddata() @@ -763,6 +764,79 @@ class Pyros(AManager): return 0 + + + # Test of the Majordome agent + def test_majordome(self): + self.changeDirectory("src") + + # Set CELERY_TEST mode ON + self.replacePatternInFile("CELERY_TEST = False", "CELERY_TEST = True", "pyros/settings.py") + self.replacePatternInFile("SIMULATOR = False", "SIMULATOR = True", "pyros/settings.py") + + # 0) Reset the database (Empty the database from any data) + self.changeDirectory("..") + self.reset_database_sim() + # Actualise l’état de la DB en accord avec l’ensemble des modèles et des migrations actuels + self.migrate() + # Load fixture initial_fixture.json + self.loaddata() + return + + # + # 1) Launch Django web server + # + ''' + self.server() + self.sleep(2) + ''' + + self.printFullTerm(Colors.WARNING, "SUMMARY") + self.printColor(Colors.GREEN, "The simulation will be ended by the task 'simulator herself'") + self.printColor(Colors.GREEN, "If you want to shutdown the simulation, please run :") + self.printColor(Colors.GREEN, "CTRL-C or pyros.py kill_simulation") + self.printColor(Colors.GREEN, "If the simulation isn't correctly killed, please switch the variable") + self.printColor(Colors.GREEN, "CELERY_TEST in src/pyros/settings.py to False") + self.printFullTerm(Colors.WARNING, "SUMMARY") + + # 2) (if USE_CELERY) Start Celery workers + # + ''' + if USE_CELERY: + self.celery_on(TOTAL) + self.sleep(3) + ''' + + # + # 3) Start simulator(s) : + # + #self.sims_launch(True) + agent='majordome' + self.changeDirectory(agent) + p = self.execProcessFromVenvAsync(self.venv_bin + ' start_agent_'+agent+'.py') + os.chdir('..') + time.sleep(5) + + + # Wait for end of simulators : + #p.wait() + + # When simulators are finished: + #self.kill_simulation() + #self.changeDirectory("src") + self.replacePatternInFile("CELERY_TEST = True", "CELERY_TEST = False", "pyros/settings.py") + self.replacePatternInFile("SIMULATOR = True", "SIMULATOR = False", "pyros/settings.py") + self.changeDirectory("..") + self.execProcessAsync("ps aux | grep \"start_agent_majordome.py\" | awk '{ print $2 }' | xargs kill") + + return 0 + + + + + + + def kill_server(self): self.execProcessAsync("fuser -k 8000/tcp") return 0 @@ -799,7 +873,7 @@ class Pyros(AManager): self.execProcessAsync("ps aux | grep \" cameraNIRSimulator.py\" | awk '{ print $2 }' | xargs kill") self.execProcessAsync("ps aux | grep \" cameraVISSimulator.py\" | awk '{ print $2 }' | xargs kill") self.changeDirectory("..") - self.stop_workers() + if USE_CELERY: self.stop_workers() self.printFullTerm(Colors.GREEN, "simulation ended") return 0 @@ -920,6 +994,7 @@ class Pyros(AManager): "start_agent_monitoring": self.start_agent_monitoring, "start_agent_majordome": self.start_agent_majordome, "start_agent_alertmanager": self.start_agent_alertmanager, + "test_majordome": self.test_majordome, # ex start "start_workers": self.start_workers, diff --git a/src/majordome/start_agent_majordome.py b/src/majordome/start_agent_majordome.py index 6ebf233..5461ee7 100755 --- a/src/majordome/start_agent_majordome.py +++ b/src/majordome/start_agent_majordome.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python3 +#!/usr/bin/env python import os import sys @@ -6,6 +6,9 @@ import sys # DJANGO setup #sys.path.append('/PROJECTS/GFT/SOFT/PYROS_SOFT/PYROS201802') sys.path.append('..') +#print("file is", __file__) +mypath = os.getcwd() + # Go into src/ os.chdir('..') #print("Current directory : " + str(os.getcwd())) diff --git a/src/majordome/start_agent_majordome_from_test.py b/src/majordome/start_agent_majordome_from_test.py new file mode 100755 index 0000000..b442f7f --- /dev/null +++ b/src/majordome/start_agent_majordome_from_test.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python + +import os +import sys + +# DJANGO setup +#sys.path.append('/PROJECTS/GFT/SOFT/PYROS_SOFT/PYROS201802') +sys.path.append('..') +#print("file is", __file__) +mypath = os.getcwd() + +# Go into src/ +##os.chdir('..') +#print("Current directory : " + str(os.getcwd())) +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "src.pyros.settings") +#os.environ['SECRET_KEY'] = 'abc' +#os.environ['ENVIRONMENT'] = 'production' +import django +django.setup() + +# MONITORING AGENT setup +agent="majordome" + +if agent=="majordome": + from majordome.tasks import Majordome + Majordome().run() diff --git a/src/majordome/tasks.py b/src/majordome/tasks.py index 08f4d1c..acc985f 100644 --- a/src/majordome/tasks.py +++ b/src/majordome/tasks.py @@ -24,6 +24,8 @@ from majordome.MajordomeDecorators import * from utils.JDManipulator import * import time +from threading import Thread + DEBUG_FILE = False log = L.setupLogger("MajordomeTaskLogger", "Majordome") @@ -35,7 +37,7 @@ log = L.setupLogger("MajordomeTaskLogger", "Majordome") check if the last schedule made has to be planned launch schedule's sequences ''' -class Majordome(Task): +class Majordome(Task, Thread): loop_speed = 1 julian_div = 86400 @@ -94,7 +96,14 @@ class Majordome(Task): # SETUP try : self.config = get_object_or_404(Config, id=1) - except Config.ObjectDoesNotExist: + #self.config = Config.objects.get(pk=1) + #self.config = Config.objects.get()[0] + + #print("maj config id is", self.config.id) + + #except Config.ObjectDoesNotExist: + except Exception as e: + print("Config read exception", str(e)) return -1 #self.config.ntc = False @@ -134,7 +143,7 @@ class Majordome(Task): if self.current_state not in ['STARTING', 'PASSIVE_NO_PLC']: if self.plc_is_not_auto(): self.changeState("PASSIVE") ''' - self.current_state = self.do_behavior_for_state(self.current_state) + self.current_state = self.do_behavior_for_current_state() ''' @@ -236,7 +245,7 @@ class Majordome(Task): print("Waiting for PLC connection") # Wait until PCL is connected while not self.plc_is_connected(): - time.sleep(2) + time.sleep(3) ''' if self.is_restarting(): self.changeState("STARTING") @@ -273,7 +282,7 @@ class Majordome(Task): if self.NEED_TO_CLOSE: self.changeState("Closing") #self.sub_behavior_closing() - self.do_behavior_for_state("Closing") + self.do_behavior_for_current_state() self.changeState("Standby") #elif self.plc_status.plc_mode != "AUTO" or self.is_shuttingdown_or_restarting(): @@ -359,8 +368,8 @@ class Majordome(Task): "Scheduler": behavior_scheduler, #"Closing": sub_behavior_closing, } - def do_behavior_for_state(self, current_state:str) -> str: - print("CURRENT OCS (MAJORDOME) STATE: "+current_state) + def do_behavior_for_current_state(self) -> str: + print("CURRENT OCS (MAJORDOME) STATE: " + self.current_state) time.sleep(2) # EXIT if PLC not connected #if not self.plc_is_connected(): return @@ -368,11 +377,11 @@ class Majordome(Task): #if self.is_shuttingdown_or_restarting(): return # run behavior for this state #self.behavior[current_state](self) - if current_state == "Closing": + if self.current_state == "Closing": self.sub_behavior_closing() else: - self.behavior[current_state](self) - return self.current_state + self.behavior[self.current_state](self) + return self.current_state ''' diff --git a/src/majordome/tests.py b/src/majordome/tests.py index c099071..56cb1c7 100644 --- a/src/majordome/tests.py +++ b/src/majordome/tests.py @@ -1,13 +1,107 @@ from django.test import TestCase +from common.models import * from .tasks import Majordome +#from django.contrib.auth.models import User +import threading +#from threading import Thread +from django.shortcuts import get_object_or_404 +import time +#import sys +import subprocess +import os + # Create your tests here. -class Majordome(TestCase): + +class MajordomeTests(TestCase): + + fixtures = ['tests/majordome_test'] + + def setUp(self): - self.majordome = Majordome() + ''' + self.m = Majordome() + self.m.run() + ''' + #Config.objects.create() + self.path_dir_file = os.path.dirname(os.path.realpath(__file__)) + #self.venv_bin = self.path_dir_file + os.sep + config["path"] + os.sep + config["env"] + os.sep + self.bin_dir + os.sep + self.bin_name + self.venv_bin = '../../private/venv_py3_pyros/bin/python' + + + def getConfigFromDB(self): + return get_object_or_404(Config, id=1) + + def execProcessAsync(self, command): + #self.printFullTerm(Colors.BLUE, "Executing command [" + command + "]") + p = subprocess.Popen(command, shell=True) + ''' + self.subproc.append((p, command)) + self.printFullTerm(Colors.GREEN, "Process launched successfully") + self.addExecuted(self.current_command, command) + ''' + return p + + def execProcessFromVenvAsync(self, command: str): + args = command.split() + #self.printFullTerm(Colors.BLUE, "Executing command from venv [" + str(' '.join(args[1:])) + "]") + p = subprocess.Popen(args) + #self.subproc.append((p, ' '.join(args[1:]))) + #self.printFullTerm(Colors.GREEN, "Process launched successfully") + #self.addExecuted(self.current_command, str(' '.join(args[1:]))) + return p + + + + def test_all(self): + ''' + self.config = get_object_or_404(Config, id=1) + print("config id is", self.config.id) + print("config latitude is", self.config.latitude) + print("config global mode is", self.config.global_mode) + print("config row_data_save_frequency is", self.config.row_data_save_frequency) + ''' + + #print_lock = threading.Lock() + + + #Majordome().run() + ''' + time.sleep(2) + thread_majordome = Majordome() + thread_majordome.start() + #thread_majordome.run() + ''' + + #with print_lock: + print("global mode is", self.getConfigFromDB().global_mode) + #sys.stdout.write("global mode") + #sys.stdout.flush() + print("global mode is", self.getConfigFromDB().global_mode) + + time.sleep(2) + #''' + + # Launch the Majordome agent + #''' + agent="majordome" + os.chdir(agent) + #print("Current directory : " + str(os.getcwd())) + p = self.execProcessFromVenvAsync(self.venv_bin + ' start_agent_'+agent+'_from_test.py') + #p = self.execProcessFromVenvAsync('./start_agent_'+agent+'.py') + #p.wait() + print("hello1") + print("hello2") + time.sleep(7) - def test_basic(self): - # TODO write some basic tests for majordome - pass \ No newline at end of file + # Kill agent Majordome + p.kill() + #self.execProcessAsync("ps aux | grep \"start_agent_majordome.py\" | awk '{ print $2 }' | xargs kill") + #self.execProcessAsync("ps aux | grep start_agent_majordome.py | awk '{ print $2 }' | xargs kill") + #''' + + print("hello") + #thread_majordome.join() + #thread_majordome.killExecutingSequence() diff --git a/src/misc/fixtures/initial_fixture.json b/src/misc/fixtures/initial_fixture.json index 904e43e..8ef57e8 100644 --- a/src/misc/fixtures/initial_fixture.json +++ b/src/misc/fixtures/initial_fixture.json @@ -1,4 +1,5 @@ [ + { "model": "common.country", "pk": 1, @@ -8,6 +9,7 @@ "quota": null } }, + { "model": "common.detector", "pk": 1, @@ -563,6 +565,7 @@ "can_del_void_req": false } }, + { "model": "common.config", "pk": 1, diff --git a/src/misc/fixtures/tests/majordome_test.json b/src/misc/fixtures/tests/majordome_test.json new file mode 100644 index 0000000..447262f --- /dev/null +++ b/src/misc/fixtures/tests/majordome_test.json @@ -0,0 +1,50 @@ +[ + +{ + "model": "common.plcdevice", + "pk": 1, + "fields": { + "name": "Plc", + "desc": "", + "created": "2018-06-18 14:32:00", + "updated": "2018-06-18 14:32:00", + "is_online": false, + "maintenance_date": null + } +}, + +{ + "model": "common.config", + "pk": 1, + "fields": { + "id": 1, + "latitude": 10, + "local_time_zone": 1, + "longitude": 1, + "altitude": 1, + "horizon_line": 1, + "row_data_save_frequency": 400, + "request_frequency": 300, + "analysed_data_save": 300, + "telescope_ip_address": "127.0.0.1", + "camera_ip_address": "127.0.0.1", + "plc_ip_address": "127.0.0.1", + "global_mode": true, + "ack": true, + "bypass": false, + "plc_timeout_seconds": 60 + } +}, + +{ + "model": "common.plcdevicestatus", + "pk": 1, + "fields": { + "id": 1, + "device_id": 1, + "created": "2018-07-18 11:49:49", + "plc_mode": "AUTO" + } +} + +] -- libgit2 0.21.2