From 4816e86b4ccffc646d41d1776a06c4b1be08a5a1 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Wed, 18 Jan 2017 15:23:11 +0100 Subject: [PATCH] removed *.sh *.bat now the manager is pyros.py --- install/install.py | 36 +++++++++++++++++++----------------- obsolete/pyrosrun.bat | 28 ++++++++++++++++++++++++++++ obsolete/pyrosrun.sh | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ obsolete/scripts/celery_test.sh | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ obsolete/scripts/delete_all_requests.sh | 19 +++++++++++++++++++ obsolete/scripts/kill_celery_workers.bat | 3 +++ obsolete/scripts/kill_celery_workers.sh | 3 +++ obsolete/scripts/kill_simulation.sh | 25 +++++++++++++++++++++++++ obsolete/scripts/start_celery_single_worker.sh | 21 +++++++++++++++++++++ obsolete/scripts/start_celery_workers.bat | 18 ++++++++++++++++++ obsolete/scripts/start_celery_workers.sh | 23 +++++++++++++++++++++++ obsolete/scripts/test_all_usual.sh | 11 +++++++++++ obsolete/sim_scripts/kill_all.sh | 4 ++++ obsolete/sim_scripts/run_all.sh | 15 +++++++++++++++ obsolete/sim_scripts/simulator_launch.sh | 147 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ obsolete/sim_scripts/start_simulator_worker.sh | 18 ++++++++++++++++++ pyros.py | 576 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ pyrosrun.bat | 28 ---------------------------- pyrosrun.sh | 100 ---------------------------------------------------------------------------------------------------- simulators/scripts/kill_all.sh | 4 ---- simulators/scripts/run_all.sh | 15 --------------- simulators/scripts/simulator_launch.sh | 147 --------------------------------------------------------------------------------------------------------------------------------------------------- simulators/scripts/start_simulator_worker.sh | 18 ------------------ simulators/user/userSimulator.py | 22 ++++++++++++++-------- src/scripts/celery_test.sh | 64 ---------------------------------------------------------------- src/scripts/delete_all_requests.sh | 19 ------------------- src/scripts/kill_celery_workers.bat | 3 --- src/scripts/kill_celery_workers.sh | 3 --- src/scripts/kill_simulation.sh | 25 ------------------------- src/scripts/start_celery_single_worker.sh | 21 --------------------- src/scripts/start_celery_workers.bat | 18 ------------------ src/scripts/start_celery_workers.sh | 23 ----------------------- src/scripts/test_all_usual.sh | 11 ----------- 33 files changed, 1108 insertions(+), 524 deletions(-) create mode 100755 obsolete/pyrosrun.bat create mode 100755 obsolete/pyrosrun.sh create mode 100755 obsolete/scripts/celery_test.sh create mode 100755 obsolete/scripts/delete_all_requests.sh create mode 100644 obsolete/scripts/kill_celery_workers.bat create mode 100755 obsolete/scripts/kill_celery_workers.sh create mode 100755 obsolete/scripts/kill_simulation.sh create mode 100755 obsolete/scripts/start_celery_single_worker.sh create mode 100644 obsolete/scripts/start_celery_workers.bat create mode 100755 obsolete/scripts/start_celery_workers.sh create mode 100755 obsolete/scripts/test_all_usual.sh create mode 100755 obsolete/sim_scripts/kill_all.sh create mode 100755 obsolete/sim_scripts/run_all.sh create mode 100755 obsolete/sim_scripts/simulator_launch.sh create mode 100755 obsolete/sim_scripts/start_simulator_worker.sh create mode 100644 pyros.py delete mode 100755 pyrosrun.bat delete mode 100755 pyrosrun.sh delete mode 100755 simulators/scripts/kill_all.sh delete mode 100755 simulators/scripts/run_all.sh delete mode 100755 simulators/scripts/simulator_launch.sh delete mode 100755 simulators/scripts/start_simulator_worker.sh delete mode 100755 src/scripts/celery_test.sh delete mode 100755 src/scripts/delete_all_requests.sh delete mode 100644 src/scripts/kill_celery_workers.bat delete mode 100755 src/scripts/kill_celery_workers.sh delete mode 100755 src/scripts/kill_simulation.sh delete mode 100755 src/scripts/start_celery_single_worker.sh delete mode 100644 src/scripts/start_celery_workers.bat delete mode 100755 src/scripts/start_celery_workers.sh delete mode 100755 src/scripts/test_all_usual.sh diff --git a/install/install.py b/install/install.py index 837e372..1304e2a 100644 --- a/install/install.py +++ b/install/install.py @@ -146,22 +146,30 @@ class AInstaller(Utils): def setConfig(self, config): self.config = config + def addExecuted(self, src, message): + if (src in self.executed): + self.executed[src].append(str(message)) + else: + self.executed[src] = [str(message)] + return 0 + + def addError(self, src, message): + if (src in self.errors): + self.errors[src].append(str(message)) + else: + self.errors[src] = [str(message)] + return 0 + def execProcess(self, command): self.printFullTerm(Colors.BLUE, "Executing command [" + command + "]") process = subprocess.Popen(command, shell=True) process.wait() if process.returncode == 0: self.printFullTerm(Colors.GREEN, "Process executed successfully") - if (self.current_command in self.executed): - self.executed[self.current_command].append(command) - else: - self.executed[self.current_command] = [command] + self.addExecuted(self.current_command, command) else: self.printFullTerm(Colors.WARNING, "Process execution failed") - if (self.current_command in self.errors): - self.errors[self.current_command].append(command) - else: - self.errors[self.current_command] = [command] + self.addError(self.current_command, command) return process.returncode def execProcessFromVenv(self, command): @@ -171,16 +179,10 @@ class AInstaller(Utils): process.wait() if process.returncode == 0: self.printFullTerm(Colors.GREEN, "Process executed successfully") - if (self.current_command in self.executed): - self.executed[self.current_command].append(str(' '.join(args[1:]))) - else: - self.executed[self.current_command] = [str(' '.join(args[1:]))] + self.addExecuted(self.current_command, str(' '.join(args[1:]))) else: self.printFullTerm(Colors.WARNING, "Process execution failed") - if (self.current_command in self.errors): - self.errors[self.current_command].append(str(' '.join(args[1:]))) - else: - self.errors[self.current_command] = [str(' '.join(args[1:]))] + self.addError(self.current_command, str(' '.join(args[1:]))) return process.returncode def askYesNoQuestion(self, message, question_tag, default=True): @@ -389,7 +391,7 @@ class Config: res = self.__parser.parse_args(), self.__parser.format_usage() return (res) except SystemExit as e: - print(e, file=sys.stderr) + # print(e, file=sys.stderr) sys.exit(1) def parseConf(self): diff --git a/obsolete/pyrosrun.bat b/obsolete/pyrosrun.bat new file mode 100755 index 0000000..2a6320e --- /dev/null +++ b/obsolete/pyrosrun.bat @@ -0,0 +1,28 @@ +@echo off + +set NEEDED_ARGUMENT=One argument is needed. Possible arguments : [runserver|test] +set INVALID_ARGUMENT=Invalid argument. Possible arguments : [runserver|test] + +private/venv_py35_pyros/Scripts/activate + +set argC=0 +for %%x in (%*) do Set /A argC+=1 + + +if %argC% NEQ 1 ( + echo %NEEDED_ARGUMENT% + pause + exit +) + +cd src/ + +if %1%==server ( + python manage.py runserver +) else if %1%==test( + python manage.py test +) else ( + echo %INVALID_ARGUMENT% +) + +pause diff --git a/obsolete/pyrosrun.sh b/obsolete/pyrosrun.sh new file mode 100755 index 0000000..4a2fa7d --- /dev/null +++ b/obsolete/pyrosrun.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +COMMANDS="\n +\t'server' : Runs the web server\n +\t'updatedb' : Update the database\n +\t'unittest' : Runs the tests that don't need celery\n +\t'test_all' : Run all the existing tests (this command needs to be updated when tests are added in the project)\n +\t'celery_on' : Starts the celery workers\n +\t'celery_off' : Stops the celery workers\n +\t'simul_on' : Starts the simulators\n +\t'simul_off' : Stops the simulators\n +\t'start' : Starts the simulators then the celery workers, then the web server\n +\t'stop' : Stops the celery workers then the simulators\n +\t'clean_logs' : Clear all pyros .log files from /logs\n +\t'simulator' : Launch routine simulator\n +\t'kill_simulation : Kill web server and celery tasks (only use during simulation)'\n +\t'sim : start only simulation task'" +#\t'test_runserver' : Launch web server on test database + +NEEDED_COMMAND="One command is needed. Possible commands : $COMMANDS" +INVALID_COMMAND="Invalid command. Possible commands : $COMMANDS" + +SCRIPT_PATH=$0 +SCRIPT_PATH=${SCRIPT_PATH%/*} + +cd $SCRIPT_PATH + +source private/venv_py3_pyros/bin/activate + +if [ $# -ne 1 ]; then + echo -e $NEEDED_COMMAND + exit +fi + +cd src/ + +case "$1" in + "server") + python manage.py runserver + ;; + "updatedb") + python manage.py makemigrations + python manage.py migrate + ;; + "unittest") + scripts/test_all_usual.sh + ;; + "test_all") + scripts/test_all_usual.sh + scripts/celery_test.sh alert_manager.tests.AlertListenerTestsCelery + # ADD HERE ALL THE CELERY TESTS + ;; + "celery_on") + scripts/start_celery_workers.sh + ;; + "celery_off") + scripts/kill_celery_workers.sh + ;; + "simul_on") + cd ../simulators + ./run_all.sh + cd - + ;; + "simul_off") + cd ../simulators + ./kill_all.sh + cd - + ;; + "start") + cd ../simulators + ./run_all.sh + cd - + scripts/start_celery_workers.sh + python manage.py runserver + ;; + "stop") + scripts/kill_celery_workers.sh + cd ../simulators + ./kill_all.sh + cd - + ;; + "clean_logs") + cd ../logs + rm -f *.log + cd - + ;; + "simulator") + ../simulators/scripts/simulator_launch.sh +# python manage.py test --pattern="testRoutine*.py" --keepdb + ;; + "kill_simulation") + ../simulators/scripts/kill_simulation.sh + ;; + "sim") + celery worker -A pyros -Q simulator_q -n pyros@simulator -c 1 & + ;; + *) + echo -e $INVALID_COMMAND + ;; +esac diff --git a/obsolete/scripts/celery_test.sh b/obsolete/scripts/celery_test.sh new file mode 100755 index 0000000..890c1e5 --- /dev/null +++ b/obsolete/scripts/celery_test.sh @@ -0,0 +1,64 @@ +# +## To be used for tests using celery (asynchronous) +## BEWARE : This script launches tests on the production database +# + +# +## Usage : celery_test.sh app.tests[.ClassTest[.test_name]] +# + +USAGE_MSG="Usage : celery_test.sh app.tests[.ClassTest[.test_name]]" +CELERY_WORKER_OFF="You must run scripts/start_celery_workers.sh before running this test" + +if [ "$#" -eq 0 ]; then + echo $USAGE_MSG + exit +fi + +# Checks if the workers are running. Useless now because we run them from the script + +# ALERT_WORKER=`ps aux | grep "celery worker" | grep alert_listener` +# if [ "${#ALERT_WORKER}" -eq 0 ]; then +# echo $CELERY_WORKER_OFF +# exit +# fi + +SCRIPT_PATH=$0 +SCRIPT_PATH=${SCRIPT_PATH%/*} + +cd $SCRIPT_PATH/.. + +# Sets the variable CELERY_TEST in settings.py in order to use to good test DB +sed -i -e "s/CELERY_TEST = False/CELERY_TEST = True/g" pyros/settings.py + +# Deletes the DB, then re-create it with basic fixtures +rm -f testdb.sqlite3 +python manage.py migrate +python manage.py loaddata misc/fixtures/initial_fixture.json + +# Starts the simulators +cd ../simulators/scripts 2>&1 /dev/null +./run_all.sh +cd - + +# Start celery +./scripts/start_celery_single_worker.sh 2>&1 /dev/null + +# Run the tests +python manage.py test "$@" --keepdb --nomigrations + +# Reset the variable in settings.py +sed -i -e "s/CELERY_TEST = True/CELERY_TEST = False/g" pyros/settings.py + +# Delete the test DB +rm -f testdb.sqlite3 + +cd scripts/ + +# Stops celery workers +./kill_celery_workers.sh + +# Stops simulators +cd ../../simulators/scripts/ +./kill_all.sh +cd - diff --git a/obsolete/scripts/delete_all_requests.sh b/obsolete/scripts/delete_all_requests.sh new file mode 100755 index 0000000..007d7d5 --- /dev/null +++ b/obsolete/scripts/delete_all_requests.sh @@ -0,0 +1,19 @@ +# This mini-script deletes all your alerts, requests, alerts, albums, plans, shs and schedules in db +# You need to have a root user with no password on your DB + +SCRIPT_PATH=$0 +SCRIPT_PATH=${SCRIPT_PATH%/*} + +cd $SCRIPT_PATH/.. + +MYSQL=`grep "MYSQL = " pyros/settings.py` +MYSQL=${MYSQL##* } +MYSQL=${MYSQL%?} + +REQUEST="delete from plan ; delete from album ; delete from sequence ; delete from alert ; delete from request ; delete from schedule_has_sequences ; delete from schedule ; delete from log" + +if [ "$MYSQL" == "True" ]; then + mysql -u root -e "$REQUEST" pyros +else + sqlite3 db.sqlite3 "$REQUEST" +fi diff --git a/obsolete/scripts/kill_celery_workers.bat b/obsolete/scripts/kill_celery_workers.bat new file mode 100644 index 0000000..5d9a89d --- /dev/null +++ b/obsolete/scripts/kill_celery_workers.bat @@ -0,0 +1,3 @@ +REM deletes all running celery workers + +taskkill /f /im celery.exe \ No newline at end of file diff --git a/obsolete/scripts/kill_celery_workers.sh b/obsolete/scripts/kill_celery_workers.sh new file mode 100755 index 0000000..aadeacd --- /dev/null +++ b/obsolete/scripts/kill_celery_workers.sh @@ -0,0 +1,3 @@ +# Mini-script to kill all the celery-workers in background + +ps aux | grep "celery worker" | awk '{print $2}' | xargs kill -9 diff --git a/obsolete/scripts/kill_simulation.sh b/obsolete/scripts/kill_simulation.sh new file mode 100755 index 0000000..e89ea5d --- /dev/null +++ b/obsolete/scripts/kill_simulation.sh @@ -0,0 +1,25 @@ +cd .. + +#Reseting +sed -i -e "s/CELERY_TEST = True/CELERY_TEST = False/g" pyros/settings.py +# sed -i -e "s/.*SIMULATOR_VALUE = .*/SIMULATOR_VALUE = None/g" userSimulator/tasks.py + +#Kill web server +fuser -k 8000/tcp + +# Delete the test DB +rm -f testdb.sqlite3 + +ps aux | grep "userSimulator.py" | awk '{ print $2 }' | xargs kill +ps aux | grep "alertSimulator.py" | awk '{ print $2 }' | xargs kill +ps aux | grep "plcSimulator.py" | awk '{ print $2 }' | xargs kill +ps aux | grep "telescopeSimulator.py" | awk '{ print $2 }' | xargs kill +ps aux | grep "cameraNIRSimulator.py" | awk '{ print $2 }' | xargs kill +ps aux | grep "cameraVISSimulator.py" | awk '{ print $2 }' | xargs kill + +cd scripts/ + +# Stops celery workers +./kill_celery_workers.sh + +echo "--------> Simulation ended <----------" diff --git a/obsolete/scripts/start_celery_single_worker.sh b/obsolete/scripts/start_celery_single_worker.sh new file mode 100755 index 0000000..b0f7f3c --- /dev/null +++ b/obsolete/scripts/start_celery_single_worker.sh @@ -0,0 +1,21 @@ +# Kills and re-launches the celery workers + +SCRIPT_PATH=$0 +SCRIPT_PATH=${SCRIPT_PATH%/*} + +cd $SCRIPT_PATH/.. + +source ../private/venv_py3_pyros/bin/activate + +./scripts/kill_celery_workers.sh +celery worker -A pyros -Q $1_q -n pyros@$1 -c 1 & +# celery worker -A pyros -Q monitoring_q -n pyros@monitoring -c 1 & +# celery worker -A pyros -Q majordome_q -n pyros@majordome -c 1 & +# celery worker -A pyros -Q scheduling_q --purge -n pyros@scheduling -c 1 & +# celery worker -A pyros -Q execute_sequence_q --purge -n pyros@execute_sequence -c 1 & +# celery worker -A pyros -Q execute_plan_vis_q --purge -n pyros@execute_plan_vis -c 1 & +# celery worker -A pyros -Q execute_plan_nir_q --purge -n pyros@execute_plan_nir -c 1 & +# celery worker -A pyros -Q create_calibrations_q --purge -n pyros@create_calibrations -c 1 & +# celery worker -A pyros -Q analysis_q --purge -n pyros@analysis -c 5 & +# celery worker -A pyros -Q system_status_q --purge -n pyros@system_status -c 1 & +# celery worker -A pyros -Q change_obs_conditions_q --purge -n pyros@change_obs_conditions -c 1 & diff --git a/obsolete/scripts/start_celery_workers.bat b/obsolete/scripts/start_celery_workers.bat new file mode 100644 index 0000000..e08cee0 --- /dev/null +++ b/obsolete/scripts/start_celery_workers.bat @@ -0,0 +1,18 @@ +REM Kills and relaunches celery workers + +REM Moves into the PYROS\src directory + +cd %~dp0% +cd .. + +call scripts\kill_celery_workers.bat + +START /B "" celery worker -A pyros -Q alert_listener_q -n pyros@alert_listener -c 1 +START /B "" celery worker -A pyros -Q scheduling_q --purge -n pyros@scheduling -c 1 +START /B "" celery worker -A pyros -Q execute_sequence_q --purge -n pyros@execute_sequence -c 1 +START /B "" celery worker -A pyros -Q execute_plan_vis_q --purge -n pyros@execute_plan_vis -c 1 +START /B "" celery worker -A pyros -Q execute_plan_nir_q --purge -n pyros@execute_plan_nir -c 1 +START /B "" celery worker -A pyros -Q analysis_q --purge -n pyros@analysis -c 5 +START /B "" celery worker -A pyros -Q system_status_q --purge -n pyros@system_status -c 1 +START /B "" celery worker -A pyros -Q change_obs_conditions_q --purge -n pyros@change_obs_conditions -c 1 +START /B "" celery worker -A pyros -Q monitoring_q -n pyros@monitoring -c 1 diff --git a/obsolete/scripts/start_celery_workers.sh b/obsolete/scripts/start_celery_workers.sh new file mode 100755 index 0000000..4f5f06a --- /dev/null +++ b/obsolete/scripts/start_celery_workers.sh @@ -0,0 +1,23 @@ +# Kills and re-launches the celery workers + +SCRIPT_PATH=$0 +SCRIPT_PATH=${SCRIPT_PATH%/*} + +cd $SCRIPT_PATH/.. + +source ../private/venv_py3_pyros/bin/activate + +./scripts/kill_celery_workers.sh +celery worker -A pyros -Q alert_listener_q -n pyros@alert_listener -c 1 & +celery worker -A pyros -Q monitoring_q -n pyros@monitoring -c 1 & +celery worker -A pyros -Q majordome_q -n pyros@majordome -c 1 & + +celery worker -A pyros -Q scheduling_q --purge -n pyros@scheduling -c 1 & +celery worker -A pyros -Q execute_sequence_q --purge -n pyros@execute_sequence -c 1 & +celery worker -A pyros -Q execute_plan_vis_q --purge -n pyros@execute_plan_vis -c 1 & +celery worker -A pyros -Q execute_plan_nir_q --purge -n pyros@execute_plan_nir -c 1 & +celery worker -A pyros -Q create_calibrations_q --purge -n pyros@create_calibrations -c 1 & +celery worker -A pyros -Q analysis_q --purge -n pyros@analysis -c 5 & +celery worker -A pyros -Q system_status_q --purge -n pyros@system_status -c 1 & +celery worker -A pyros -Q change_obs_conditions_q --purge -n pyros@change_obs_conditions -c 1 & + diff --git a/obsolete/scripts/test_all_usual.sh b/obsolete/scripts/test_all_usual.sh new file mode 100755 index 0000000..2b46698 --- /dev/null +++ b/obsolete/scripts/test_all_usual.sh @@ -0,0 +1,11 @@ +# +## Launches all the tests not needing comet or celery +# + + +SCRIPT_PATH=$0 +SCRIPT_PATH=${SCRIPT_PATH%/*} + +cd $SCRIPT_PATH/.. + +../private/venv_py3_pyros/bin/python3 manage.py test common scheduler routine_manager user_manager alert_manager.tests.TestStrategyChange diff --git a/obsolete/sim_scripts/kill_all.sh b/obsolete/sim_scripts/kill_all.sh new file mode 100755 index 0000000..d9245d8 --- /dev/null +++ b/obsolete/sim_scripts/kill_all.sh @@ -0,0 +1,4 @@ +ps aux | grep "python ../telescope/Telescope.py" | awk '{print $2}' | xargs kill -9 +ps aux | grep "python ../camera/CameraVIS.py" | awk '{print $2}' | xargs kill -9 +ps aux | grep "python ../camera/CameraNIR.py" | awk '{print $2}' | xargs kill -9 +ps aux | grep "python ../plc/PLC.py" | awk '{print $2}' | xargs kill -9 diff --git a/obsolete/sim_scripts/run_all.sh b/obsolete/sim_scripts/run_all.sh new file mode 100755 index 0000000..665356a --- /dev/null +++ b/obsolete/sim_scripts/run_all.sh @@ -0,0 +1,15 @@ +SCRIPT_PATH=$0 +SCRIPT_PATH=${SCRIPT_PATH%/*} + +./kill_all.sh + +cd ../../src/ + +source ../private/venv_py3_pyros/bin/activate + +cd - + +python ../telescope/Telescope.py & +python ../camera/CameraVIS.py & +python ../camera/CameraNIR.py & +python ../plc/PLC.py & diff --git a/obsolete/sim_scripts/simulator_launch.sh b/obsolete/sim_scripts/simulator_launch.sh new file mode 100755 index 0000000..119bcf6 --- /dev/null +++ b/obsolete/sim_scripts/simulator_launch.sh @@ -0,0 +1,147 @@ +# setting + +SIM_FIXTURES="../simulators/config" +SIM_PATH="../simulators/" + +cd "$SIM_PATH"config/ +echo -n "Existing simulations : " +ls conf*.json +read -e -p "Enter which simulation you want to launch : " line +cd - > /dev/null + +if [[ $line != *[!\ ]* ]]; then + line="simulation.json" +fi + +if [ ! -f "$SIM_FIXTURES"/"$line" ]; then + echo "This simulation doesn't exist" + exit +fi + +# sed -i -e "s/SIMULATOR_VALUE = None/SIMULATOR_VALUE = '$line'/g" userSimulator/tasks.py +sed -i -e "s/CELERY_TEST = False/CELERY_TEST = True/g" pyros/settings.py + +trap ctrl_c INT + +function kill_sim() +{ + #Reseting + sed -i -e "s/CELERY_TEST = True/CELERY_TEST = False/g" pyros/settings.py + # sed -i -e "s/.*SIMULATOR_VALUE = .*/SIMULATOR_VALUE = None/g" userSimulator/tasks.py + + #Kill web server + fuser -k 8000/tcp + + # Delete the test DB + rm -f testdb.sqlite3 + + ps aux | grep "userSimulator.py" | awk '{ print $2 }' | xargs kill + ps aux | grep "alertSimulator.py" | awk '{ print $2 }' | xargs kill + ps aux | grep "plcSimulator.py" | awk '{ print $2 }' | xargs kill + ps aux | grep "telescopeSimulator.py" | awk '{ print $2 }' | xargs kill + ps aux | grep "cameraNIRSimulator.py" | awk '{ print $2 }' | xargs kill + ps aux | grep "cameraVISSimulator.py" | awk '{ print $2 }' | xargs kill + + cd scripts/ + + # Stops celery workers + ./kill_celery_workers.sh + + echo "--------> Simulation ended <----------" +} + +function ctrl_c() { + kill_sim + echo "Ctrl-c catched, all process launched have been killed" + exit +} + +rm -f testdb.sqlite3 +python manage.py migrate +python manage.py loaddata misc/fixtures/initial_fixture.json + +# python manage.py loaddata "$SIM_FIXTURES"/"$line" + +echo "----------------LAUNCHING WEB SERVER----------------" +python manage.py runserver > /dev/null 2>&1 & + +sleep 2 +echo "----------------WEB SERVER LAUNCHED-----------------" + +echo "----------------------------------------SUMMARY---------------------------------------------" +echo "The simulator has been successfully initialised" +echo "The simulator run on a temp database : src/testdb.sqlite3" +echo "The simulation will be ended by the task 'simulator herself'" +echo "If you want to shutdown the simulation, please run :" +echo "CTRL-C or ./pyrosrun.sh kill_simulation" +echo "If the simulation isn't correctly killed, please switch the variable" +echo "CELERY_TEST in src/pyros/settings.py to false" +echo "" +echo "----------------------------------------SUMMARY---------------------------------------------" +echo "" +read -n1 -r -p "Press any key to continue..." key + +#launch simulator workers +./../simulators/scripts/start_simulator_worker.sh 2>&1 /dev/null + +sleep 2 + +#launch simulators binaries +cd "$SIM_PATH"user/ +python userSimulator.py "$line" & +USERPID=$! +cd - +cd "$SIM_PATH"alert/ +python alertSimulator.py "$line" & +ALERTPID=$! +cd - +cd "$SIM_PATH"plc/ +python plcSimulator.py "$line" & +PLCPID=$! +cd - +cd "$SIM_PATH"camera/ +python cameraVISSimulator.py "$line" & +CAMERAVISPID=$! +python cameraNIRSimulator.py "$line" & +CAMERANIRPID=$! +cd - +cd "$SIM_PATH"telescope/ +python telescopeSimulator.py "$line" & +TELESCOPEPID=$! +cd - + +echo "" +echo "Waiting for userSimulator to finish" +wait "$USERPID" +echo "userSimulator finished" +echo "" + +echo "" +echo "Waiting for alertSimulator to finish" +wait "$ALERTPID" +echo "alertSimulator finished" +echo "" + +echo "Waiting for plcSimulator to finish" +wait "$PLCPID" +echo "plcSimulator finished" +echo "" + +echo "Waiting for telescopeSimulator to finish" +wait "$PLCPID" +echo "telescopeSimulator finished" +echo "" + +echo "Waiting for cameraNIRSimulator to finish" +wait "$CAMERANIRPID" +echo "cameraNIRSimulator finished" +echo "" + +echo "Waiting for cameraVISSimulator to finish" +wait "$CAMERAVISPID" +echo "cameraVISSimulator finished" +echo "" + +echo "Killing all processes" + +kill_sim \ No newline at end of file diff --git a/obsolete/sim_scripts/start_simulator_worker.sh b/obsolete/sim_scripts/start_simulator_worker.sh new file mode 100755 index 0000000..a356d40 --- /dev/null +++ b/obsolete/sim_scripts/start_simulator_worker.sh @@ -0,0 +1,18 @@ +# Kill and re-launch simulator workers + +source ../private/venv_py3_pyros/bin/activate + +./scripts/kill_celery_workers.sh + +celery worker -A pyros -Q monitoring_q -n pyros@monitoring -c 1 & +celery worker -A pyros -Q alert_listener_q -n pyros@alert_listener -c 1 & +celery worker -A pyros -Q majordome_q -n pyros@majordome -c 1 & + +celery worker -A pyros -Q scheduling_q --purge -n pyros@scheduling -c 1 & +celery worker -A pyros -Q execute_sequence_q --purge -n pyros@execute_sequence -c 1 & +celery worker -A pyros -Q execute_plan_vis_q --purge -n pyros@execute_plan_vis -c 1 & +celery worker -A pyros -Q execute_plan_nir_q --purge -n pyros@execute_plan_nir -c 1 & +celery worker -A pyros -Q create_calibrations_q --purge -n pyros@create_calibrations -c 1 & +celery worker -A pyros -Q analysis_q --purge -n pyros@analysis -c 5 & +celery worker -A pyros -Q system_status_q --purge -n pyros@system_status -c 1 & +celery worker -A pyros -Q change_obs_conditions_q --purge -n pyros@change_obs_conditions -c 1 & diff --git a/pyros.py b/pyros.py new file mode 100644 index 0000000..80fa30d --- /dev/null +++ b/pyros.py @@ -0,0 +1,576 @@ +import sys +import os +import subprocess +import platform +import fileinput +import argparse +import time + +DEBUG = False + + +class Utils: + system = platform.system() + columns = 100 + row = 1000 + disp = True + + def __init__(self): + if (platform.system() != 'Windows'): + try: + rows, columns = os.popen('stty size', 'r').read().split() + self.columns = int(columns) + except: + self.columns = 100 + if DEBUG: + print("Could not get terminal size") + + def printFullTerm(self, color, string): + value = int(self.columns / 2 - len(string) / 2) + self.printColor(color, "-" * value, eol='') + self.printColor(color, string, eol='') + value += len(string) + self.printColor(color, "-" * (self.columns - value)) + return 0 + + def changeDirectory(self, path): + if DEBUG: + print("Moving to : " + path) + os.chdir(path) + if DEBUG: + print("Current directory : " + str(os.getcwd())) + return 0 + + def replacePatternInFile(self, pattern, replace, file_path): + try: + with fileinput.FileInput(file_path, inplace=True, backup='.bak') as file: + for line in file: + print(line.replace(pattern, replace), end='') + except: + return 1 + return 0 + + def printColor(self, color, message, file=sys.stdout, eol=os.linesep, forced=False): + if (self.disp == False and forced == False): + return 0 + if (self.system == 'Windows'): + print(message, file=file, end=eol) + else: + print(color + message + Colors.ENDC, file=file, end=eol) + return 0 + + def askQuestion(self, message, default = ""): + self.printColor(Colors.BLUE, message, forced=True) + self.printColor(Colors.BOLD, "Answer (default="+default+"): ", eol='', forced=True) + sys.stdout.flush() + ret = sys.stdin.readline().replace(os.linesep, '') + if ret == "": + return default + return ret + + def sleep(self, t): + time.sleep(t) + return 0 + +''' + Manager class : manager of your project +''' + + +class AManager(Utils): + path = os.path.realpath(__file__) + path_dir = os.getcwd() + path_dir_file = os.path.dirname(os.path.realpath(__file__)) + python_path = sys.executable + python_version = sys.version_info + + bin_dir = "" + celery = "celery" + venv_pip = "pip" + venv_bin = "python" + wait = True + current_command = "" + config = None + commandMatcher = {} + commandDescription = {} + commands = [] + errors = {} + executed = {} + + def __init__(self, param): + super(AManager, self).__init__() + self.wait = param.getWait() + self.commands = param.getCommandList() + self.disp = param.getPrint() + config = param.getConfig() + self.config = config + + self.changeDirectory(self.path_dir_file) + + if self.system == 'Windows': + self.bin_dir = "Scripts" + self.bin_name = "python.exe" + self.pip_name = "pip.exe" + self.celery = "celery.exe" + else: + self.bin_dir = "bin" + self.bin_name = "python" + self.pip_name = "pip" + self.celery = "celery" + self.venv_pip = self.path_dir_file + os.sep + config["path"] + os.sep + config["env"] + os.sep + self.bin_dir + os.sep + self.pip_name + self.venv_bin = self.path_dir_file + os.sep + config["path"] + os.sep + config["env"] + os.sep + self.bin_dir + os.sep + self.bin_name + self.venv_cel = self.path_dir_file + os.sep + config["path"] + os.sep + config["env"] + os.sep + self.bin_dir + os.sep + self.celery + + def help(self): + print("This function must be implemented") + raise(NotImplementedError("Function not implemented")) + + def addExecuted(self, src, message): + if (src in self.executed): + self.executed[src].append(str(message)) + else: + self.executed[src] = [str(message)] + return 0 + + def addError(self, src, message): + if (src in self.errors): + self.errors[src].append(str(message)) + else: + self.errors[src] = [str(message)] + return 0 + + def execProcess(self, command): + self.printFullTerm(Colors.BLUE, "Executing command [" + command + "]") + process = subprocess.Popen(command, shell=True) + process.wait() + if process.returncode == 0: + self.printFullTerm(Colors.GREEN, "Process executed successfully") + self.addExecuted(self.current_command, command) + else: + self.printFullTerm(Colors.WARNING, "Process execution failed") + self.addError(self.current_command, command) + return process.returncode + + def execProcessFromVenv(self, command): + args = command.split() + self.printFullTerm(Colors.BLUE, "Executing command from venv [" + str(' '.join(args[1:])) + "]") + process = subprocess.Popen(args) + process.wait() + if process.returncode == 0: + self.printFullTerm(Colors.GREEN, "Process executed successfully") + self.addExecuted(self.current_command, str(' '.join(args[1:]))) + else: + self.printFullTerm(Colors.WARNING, "Process execution failed") + self.addError(self.current_command, str(' '.join(args[1:]))) + return process.returncode + + def execProcessAsync(self, command): + self.printFullTerm(Colors.BLUE, "Executing command [" + command + "]") + subprocess.Popen(command, shell=True) + self.printFullTerm(Colors.GREEN, "Process launched successfully") + self.addExecuted(self.current_command, command) + return 0 + + def execProcessFromVenvAsync(self, command: str) -> int: + args = command.split() + self.printFullTerm(Colors.BLUE, "Executing command from venv [" + str(' '.join(args[1:])) + "]") + subprocess.Popen(args) + self.printFullTerm(Colors.GREEN, "Process launched successfully") + self.addExecuted(self.current_command, str(' '.join(args[1:]))) + return 0 + + def end(self): + count = 0 + + self.printFullTerm(Colors.WARNING, "Summary") + self.printColor(Colors.GREEN, "Success : ") + for command, valid in self.executed.items(): + if not valid: + self.printColor(Colors.BLUE, "\t- Command : " + command + " success !") + else: + self.printColor(Colors.WARNING, "\t- In commmand : " + command) + for exe in valid: + self.printColor(Colors.GREEN, "\t\t - Command : " + exe + " success !") + self.printColor(Colors.FAIL, "Errors : ") + if not self.errors: + self.printColor(Colors.GREEN, "\tNone") + for command, items in self.errors.items(): + count += 1 + if (not items): + self.printColor(Colors.FAIL, "Command : " + command + " failed !") + else: + self.printColor(Colors.WARNING, "\t- In commmand : " + command) + for exe in items: + self.printColor(Colors.FAIL, "\t\t - Command : " + exe) + return count + + def exec(self): + if (not self.commands): + self.commandMatcher["help"]() + return 0 + for command in self.commands: + self.current_command = command + if command in self.commandMatcher: + self.commandMatcher[command]() + else: + self.addError(str(command), "invalid command") + return self.end() + + def logError(self, message): + self.printColor(Colors.FAIL, "Pyros : An error occurred [" + message + "]", file=sys.stderr) + return 0 + + +class Config: + __parser = argparse.ArgumentParser("Installer parser") + __content = { + "path": "private", + "env": "venv_py3_pyros" + } + __wait = True + __print = True + usage = "" + __command_list = [] + + def __init__(self): + self.__parser.add_argument("command", nargs='?', default="help", help="The command you want to execute (default=help)") + self.__parser.add_argument("--env", help="Your environment directory name default=venv") + self.__parser.add_argument("--path", help="Path to the virtual env (from the source file directory) (default=private)") + self.__parser.add_argument("--nowait", action='store_true', help="Don't wait the end of a program") + self.__parser.add_argument("--noprint", action='store_true', help="Won't print") + self.usage = self.__parser.format_usage() + + def parse(self): + try: + res = self.__parser.parse_args(), self.__parser.format_usage() + return (res) + except SystemExit as e: + # print(e, file=sys.stderr) + sys.exit(1) + + def parseConf(self): + res, usage = self.parse() + try: + if (res.env): + self.__content["env"] = res.env + if (res.path): + self.__content["path"] = res.path + if (res.nowait): + self.__wait = False + if (res.noprint): + self.__print = False + self.__command_list.append(res.command) + return 0 + except Exception as e: + print(e, file=sys.stderr) + return 1 + + def getPath(self): + return self.__content["path"] + + def getEnv(self): + return self.__content["env"] + + def getWait(self): + return self.__wait + + def getPrint(self): + return self.__print + + def setPrint(self, value): + self.__print = value + return 0 + + def setWait(self, value): + self.__wait = value + return 0 + + def getCommandList(self): + return self.__command_list + + def printUsage(self): + print(self.usage, file=sys.stderr) + + def addConf(self, key, value): + if isinstance(key, str) and isinstance(value, str): + self.__content[key] = value + return 0 + return 1 + + def setPath(self, path): + if (os.path.isdir(path)): + if (path == ""): + path = "." + self.__content["path"] = path + return 0 + return 1 + + def setEnvName(self, name): + self.__content["env"] = name + return 0 + + def getConfig(self): + return self.__content + + +''' + Color class +''' + + +class Colors: + HEADER = '\033[95m' + BLUE = '\033[94m' + GREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + + +''' + Pyros class +''' + + +class Pyros(AManager): + help_message = "python neo.py" + init_fixture = "initial_fixture.json" + + def install(self): + if (self.system == "Windows"): + self.execProcess("python install/install.py install") + else: + self.execProcess("python3 install/install.py install") + return 0 + + def update(self): + if (self.system == "Windows"): + self.execProcess("python install/install.py update") + else: + self.execProcess("python3 install/install.py update") + return 0 + + def server(self): + self.changeDirectory("src") + self.execProcessFromVenvAsync(self.venv_bin + " manage.py runserver") + self.changeDirectory("..") + return 0 + + def clean(self): + return self.clean_logs() + + def clean_logs(self): + return self.execProcess("rm logs/*.log") + + def test(self): + self.changeDirectory("src") + self.execProcessFromVenvAsync(self.venv_bin + " manage.py test") + self.changeDirectory("..") + return 0 + + def migrate(self): + self.changeDirectory("src") + self.execProcessFromVenv(self.venv_bin + " manage.py migrate") + self.changeDirectory("..") + return 0 + + def makemigrations(self): + self.changeDirectory("src") + self.execProcessFromVenv(self.venv_bin + " manage.py makemigrations") + self.changeDirectory("..") + return 0 + + def help(self): + for command, message in self.commandDescription.items(): + self.printColor(Colors.BLUE, "-> " + command + ": ", eol='') + self.printColor(Colors.GREEN, message) + return 0 + + def updatedb(self): + self.migrate() + self.makemigrations() + return 0 + + def unittest(self): + self.changeDirectory("src") + self.execProcessFromVenv(self.venv_bin + " manage.py test common scheduler routine_manager user_manager alert_manager.tests.TestStrategyChange") + self.changeDirectory("..") + return 0 + + def test_all(self): + self.unittest() + + self.changeDirectory("src") + self.replacePatternInFile("CELERY_TEST = False", "CELERY_TEST = True", "pyros/settings.py") + self.execProcess("rm -f testdb.sqlite3") + self.changeDirectory("..") + + self.migrate() + self.loaddata() + self.sims_launch() + self.singleWorker("alert_manager") + self.changeDirectory("src") + self.execProcessFromVenv(self.venv_bin + " manage.py test alert_manager.tests.AlertListenerTestsCelery --keepdb --nomigrations") + self.replacePatternInFile("CELERY_TEST = True", "CELERY_TEST = False", "pyros/settings.py") + self.execProcess("rm -f testdb.sqlite3") + self.changeDirectory("..") + self.kill_simulation() + return 0 + + def loaddata(self): + self.changeDirectory("src") + self.execProcessFromVenv(self.venv_bin + " manage.py loaddata misc/fixtures/" + self.init_fixture) + self.changeDirectory("..") + return 0 + + def celery_on(self): + self.changeDirectory("src") + self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q alert_listener_q -n pyros@alert_listener -c 1") + self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q monitoring_q -n pyros@monitoring -c 1") + self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q majordome_q -n pyros@majordome -c 1") + + self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q scheduling_q --purge -n pyros@scheduling -c 1") + self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q execute_sequence_q --purge -n pyros@execute_sequence -c 1") + self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q execute_plan_vis_q --purge -n pyros@execute_plan_vis -c 1") + self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q execute_plan_vis_q --purge -n pyros@execute_plan_nir -c 1") + self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q create_calibrations_q --purge -n pyros@create_calibrations -c 1") + self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q analysis_q --purge -n pyros@analysis -c 1") + self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q system_status_q --purge -n pyros@system_status -c 1") + self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q change_obs_conditions_q --purge -n pyros@change_obs_conditions -c 1") + self.changeDirectory("..") + return 0 + + def start(self): + self.stop() + self.celery_on() + return 0 + + def stop(self): + self.execProcessAsync("ps aux | grep \"celery worker\" | awk '{print $2}' | xargs kill -9") + return 0 + + def simulator(self): + self.changeDirectory("src") + self.replacePatternInFile("CELERY_TEST = True", "CELERY_TEST = False", "pyros/settings.py") + self.execProcess("rm -f testdb.sqlite3") + self.changeDirectory("..") + self.migrate() + self.loaddata() + self.server() + self.sleep(2) + self.printFullTerm(Colors.WARNING, "SUMMARY") + self.printColor(Colors.GREEN, "The simulator has been successfully initialised") + self.printColor(Colors.GREEN, "The simulator run on a temp database : src/testdb.sqlite3") + self.printColor(Colors.GREEN, "The simulation will be ended by the task 'simulator herself'") + self.printColor(Colors.GREEN, "If you want to shutdown the simulation, please run :") + self.printColor(Colors.GREEN, "CTRL-C or ./pyrosrun.sh kill_simulation") + self.printColor(Colors.GREEN, "If the simulation isn't correctly killed, please switch the variable") + self.printColor(Colors.GREEN, "CELERY_TEST in src/pyros/settings.py to false") + self.printFullTerm(Colors.WARNING, "SUMMARY") + self.celery_on() + self.sleep(3) + self.sims_launch() + return 0 + + def kill_simulation(self): + self.changeDirectory("src") + self.replacePatternInFile("CELERY_TEST = True", "CELERY_TEST = False", "pyros/settings.py") + self.execProcessAsync("fuser -k 8000/tcp") + self.execProcessAsync("rm -f testdb.sqlite3") + self.execProcessAsync("ps aux | grep \" userSimulator.py\" | awk '{ print $2 }' | xargs kill") + self.execProcessAsync("ps aux | grep \" alertSimulator.py\" | awk '{ print $2 }' | xargs kill") + self.execProcessAsync("ps aux | grep \" plcSimulator.py\" | awk '{ print $2 }' | xargs kill") + self.execProcessAsync("ps aux | grep \" telescopeSimulator.py\" | awk '{ print $2 }' | xargs kill") + self.execProcessAsync("ps aux | grep \" cameraNIRSimulator.py\" | awk '{ print $2 }' | xargs kill") + self.execProcessAsync("ps aux | grep \" cameraVISSimulator.py\" | awk '{ print $2 }' | xargs kill") + self.changeDirectory("..") + self.stop() + self.printFullTerm(Colors.GREEN, "simulation ended") + return 0 + + def sims_launch(self): + self.changeDirectory("simulators/config") + self.execProcess("ls conf*.json") + conf = self.askQuestion("Which simulation do you want to use", default="conf.json") + if not os.path.isfile(conf): + self.printColor(Colors.FAIL, "The simulation file " + conf + " does not exist") + return 1 + self.changeDirectory("..") + self.changeDirectory("user") + self.execProcessFromVenvAsync(self.venv_bin + " userSimulator.py " + conf) + self.changeDirectory("..") + self.changeDirectory("alert") + self.execProcessFromVenvAsync(self.venv_bin + " alertSimulator.py " + conf) + self.changeDirectory("..") + self.changeDirectory("plc") + self.execProcessFromVenvAsync(self.venv_bin + " plcSimulator.py " + conf) + self.changeDirectory("..") + self.changeDirectory("camera") + self.execProcessFromVenvAsync(self.venv_bin + " cameraVISSimulator.py " + conf) + self.execProcessFromVenvAsync(self.venv_bin + " cameraNIRSimulator.py " + conf) + self.changeDirectory("..") + self.changeDirectory("telescope") + self.execProcessFromVenvAsync(self.venv_bin + " telescopeSimulator.py " + conf) + self.changeDirectory("..") + self.changeDirectory("..") + return 0 + + def singleWorker(self, worker): + self.changeDirectory("src") + self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q "+ worker +"_q -n pyros@"+worker+" -c 1") + self.changeDirectory("..") + return 0 + + def __init__(self, argv): + super(Pyros, self).__init__(argv) + self.commandMatcher = { + "install": self.install, + "update": self.update, + "server": self.server, + "clean": self.clean, + "clean_logs": self.clean_logs, + "test": self.test, + "migrate": self.migrate, + "makemigrations": self.makemigrations, + "updatedb": self.updatedb, + "unittest": self.unittest, + "test_all": self.test_all, + "celery_on": self.celery_on, + "loaddata": self.loaddata, + "start": self.start, + "stop": self.stop, + "simulator": self.simulator, + "kill_simulation": self.kill_simulation, + "sims_launch": self.sims_launch, + "help": self.help, + } + self.commandDescription = { + "install": "Launch the server installation", + "update": "Update the server", + "server": "Launch the web server", + "loaddata": "Load the initial fixture in database", + "clean": "clean the repository", + "clean_logs": "clean the log directory", + "test": "launch the server tests", + "migrate": "execute migrations", + "makemigrations": "create new migrations", + "help": "Help message", + "updatedb": "Update the database", + "unittest": "Runs the tests thaht don't need celery", + "test_all": "Run all the existing tests (this command needs to be updated when tests are added in the project", + "celery_on": "Starts celery workers", + "start": "Stop the celery workers then the web server", + "stop": "stops the celery workers", + "simulator": "Launch a simulation", + "kill_simulation": "kill the simulators / celery workers / web server", + "sims_launch": "Launch only the simulators", + } + +if __name__ == "__main__": + conf = Config() + if conf.parseConf(): + sys.exit(1) + pyros = Pyros(conf) + sys.exit(pyros.exec()) \ No newline at end of file diff --git a/pyrosrun.bat b/pyrosrun.bat deleted file mode 100755 index 2a6320e..0000000 --- a/pyrosrun.bat +++ /dev/null @@ -1,28 +0,0 @@ -@echo off - -set NEEDED_ARGUMENT=One argument is needed. Possible arguments : [runserver|test] -set INVALID_ARGUMENT=Invalid argument. Possible arguments : [runserver|test] - -private/venv_py35_pyros/Scripts/activate - -set argC=0 -for %%x in (%*) do Set /A argC+=1 - - -if %argC% NEQ 1 ( - echo %NEEDED_ARGUMENT% - pause - exit -) - -cd src/ - -if %1%==server ( - python manage.py runserver -) else if %1%==test( - python manage.py test -) else ( - echo %INVALID_ARGUMENT% -) - -pause diff --git a/pyrosrun.sh b/pyrosrun.sh deleted file mode 100755 index 4a2fa7d..0000000 --- a/pyrosrun.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/bin/bash - -COMMANDS="\n -\t'server' : Runs the web server\n -\t'updatedb' : Update the database\n -\t'unittest' : Runs the tests that don't need celery\n -\t'test_all' : Run all the existing tests (this command needs to be updated when tests are added in the project)\n -\t'celery_on' : Starts the celery workers\n -\t'celery_off' : Stops the celery workers\n -\t'simul_on' : Starts the simulators\n -\t'simul_off' : Stops the simulators\n -\t'start' : Starts the simulators then the celery workers, then the web server\n -\t'stop' : Stops the celery workers then the simulators\n -\t'clean_logs' : Clear all pyros .log files from /logs\n -\t'simulator' : Launch routine simulator\n -\t'kill_simulation : Kill web server and celery tasks (only use during simulation)'\n -\t'sim : start only simulation task'" -#\t'test_runserver' : Launch web server on test database - -NEEDED_COMMAND="One command is needed. Possible commands : $COMMANDS" -INVALID_COMMAND="Invalid command. Possible commands : $COMMANDS" - -SCRIPT_PATH=$0 -SCRIPT_PATH=${SCRIPT_PATH%/*} - -cd $SCRIPT_PATH - -source private/venv_py3_pyros/bin/activate - -if [ $# -ne 1 ]; then - echo -e $NEEDED_COMMAND - exit -fi - -cd src/ - -case "$1" in - "server") - python manage.py runserver - ;; - "updatedb") - python manage.py makemigrations - python manage.py migrate - ;; - "unittest") - scripts/test_all_usual.sh - ;; - "test_all") - scripts/test_all_usual.sh - scripts/celery_test.sh alert_manager.tests.AlertListenerTestsCelery - # ADD HERE ALL THE CELERY TESTS - ;; - "celery_on") - scripts/start_celery_workers.sh - ;; - "celery_off") - scripts/kill_celery_workers.sh - ;; - "simul_on") - cd ../simulators - ./run_all.sh - cd - - ;; - "simul_off") - cd ../simulators - ./kill_all.sh - cd - - ;; - "start") - cd ../simulators - ./run_all.sh - cd - - scripts/start_celery_workers.sh - python manage.py runserver - ;; - "stop") - scripts/kill_celery_workers.sh - cd ../simulators - ./kill_all.sh - cd - - ;; - "clean_logs") - cd ../logs - rm -f *.log - cd - - ;; - "simulator") - ../simulators/scripts/simulator_launch.sh -# python manage.py test --pattern="testRoutine*.py" --keepdb - ;; - "kill_simulation") - ../simulators/scripts/kill_simulation.sh - ;; - "sim") - celery worker -A pyros -Q simulator_q -n pyros@simulator -c 1 & - ;; - *) - echo -e $INVALID_COMMAND - ;; -esac diff --git a/simulators/scripts/kill_all.sh b/simulators/scripts/kill_all.sh deleted file mode 100755 index d9245d8..0000000 --- a/simulators/scripts/kill_all.sh +++ /dev/null @@ -1,4 +0,0 @@ -ps aux | grep "python ../telescope/Telescope.py" | awk '{print $2}' | xargs kill -9 -ps aux | grep "python ../camera/CameraVIS.py" | awk '{print $2}' | xargs kill -9 -ps aux | grep "python ../camera/CameraNIR.py" | awk '{print $2}' | xargs kill -9 -ps aux | grep "python ../plc/PLC.py" | awk '{print $2}' | xargs kill -9 diff --git a/simulators/scripts/run_all.sh b/simulators/scripts/run_all.sh deleted file mode 100755 index 665356a..0000000 --- a/simulators/scripts/run_all.sh +++ /dev/null @@ -1,15 +0,0 @@ -SCRIPT_PATH=$0 -SCRIPT_PATH=${SCRIPT_PATH%/*} - -./kill_all.sh - -cd ../../src/ - -source ../private/venv_py3_pyros/bin/activate - -cd - - -python ../telescope/Telescope.py & -python ../camera/CameraVIS.py & -python ../camera/CameraNIR.py & -python ../plc/PLC.py & diff --git a/simulators/scripts/simulator_launch.sh b/simulators/scripts/simulator_launch.sh deleted file mode 100755 index 119bcf6..0000000 --- a/simulators/scripts/simulator_launch.sh +++ /dev/null @@ -1,147 +0,0 @@ -# setting - -SIM_FIXTURES="../simulators/config" -SIM_PATH="../simulators/" - -cd "$SIM_PATH"config/ -echo -n "Existing simulations : " -ls conf*.json -read -e -p "Enter which simulation you want to launch : " line -cd - > /dev/null - -if [[ $line != *[!\ ]* ]]; then - line="simulation.json" -fi - -if [ ! -f "$SIM_FIXTURES"/"$line" ]; then - echo "This simulation doesn't exist" - exit -fi - -# sed -i -e "s/SIMULATOR_VALUE = None/SIMULATOR_VALUE = '$line'/g" userSimulator/tasks.py -sed -i -e "s/CELERY_TEST = False/CELERY_TEST = True/g" pyros/settings.py - -trap ctrl_c INT - -function kill_sim() -{ - #Reseting - sed -i -e "s/CELERY_TEST = True/CELERY_TEST = False/g" pyros/settings.py - # sed -i -e "s/.*SIMULATOR_VALUE = .*/SIMULATOR_VALUE = None/g" userSimulator/tasks.py - - #Kill web server - fuser -k 8000/tcp - - # Delete the test DB - rm -f testdb.sqlite3 - - ps aux | grep "userSimulator.py" | awk '{ print $2 }' | xargs kill - ps aux | grep "alertSimulator.py" | awk '{ print $2 }' | xargs kill - ps aux | grep "plcSimulator.py" | awk '{ print $2 }' | xargs kill - ps aux | grep "telescopeSimulator.py" | awk '{ print $2 }' | xargs kill - ps aux | grep "cameraNIRSimulator.py" | awk '{ print $2 }' | xargs kill - ps aux | grep "cameraVISSimulator.py" | awk '{ print $2 }' | xargs kill - - cd scripts/ - - # Stops celery workers - ./kill_celery_workers.sh - - echo "--------> Simulation ended <----------" -} - -function ctrl_c() { - kill_sim - echo "Ctrl-c catched, all process launched have been killed" - exit -} - -rm -f testdb.sqlite3 -python manage.py migrate -python manage.py loaddata misc/fixtures/initial_fixture.json - -# python manage.py loaddata "$SIM_FIXTURES"/"$line" - -echo "----------------LAUNCHING WEB SERVER----------------" -python manage.py runserver > /dev/null 2>&1 & - -sleep 2 -echo "----------------WEB SERVER LAUNCHED-----------------" - -echo "----------------------------------------SUMMARY---------------------------------------------" -echo "The simulator has been successfully initialised" -echo "The simulator run on a temp database : src/testdb.sqlite3" -echo "The simulation will be ended by the task 'simulator herself'" -echo "If you want to shutdown the simulation, please run :" -echo "CTRL-C or ./pyrosrun.sh kill_simulation" -echo "If the simulation isn't correctly killed, please switch the variable" -echo "CELERY_TEST in src/pyros/settings.py to false" -echo "" -echo "----------------------------------------SUMMARY---------------------------------------------" -echo "" -read -n1 -r -p "Press any key to continue..." key - -#launch simulator workers -./../simulators/scripts/start_simulator_worker.sh 2>&1 /dev/null - -sleep 2 - -#launch simulators binaries -cd "$SIM_PATH"user/ -python userSimulator.py "$line" & -USERPID=$! -cd - -cd "$SIM_PATH"alert/ -python alertSimulator.py "$line" & -ALERTPID=$! -cd - -cd "$SIM_PATH"plc/ -python plcSimulator.py "$line" & -PLCPID=$! -cd - -cd "$SIM_PATH"camera/ -python cameraVISSimulator.py "$line" & -CAMERAVISPID=$! -python cameraNIRSimulator.py "$line" & -CAMERANIRPID=$! -cd - -cd "$SIM_PATH"telescope/ -python telescopeSimulator.py "$line" & -TELESCOPEPID=$! -cd - - -echo "" -echo "Waiting for userSimulator to finish" -wait "$USERPID" -echo "userSimulator finished" -echo "" - -echo "" -echo "Waiting for alertSimulator to finish" -wait "$ALERTPID" -echo "alertSimulator finished" -echo "" - -echo "Waiting for plcSimulator to finish" -wait "$PLCPID" -echo "plcSimulator finished" -echo "" - -echo "Waiting for telescopeSimulator to finish" -wait "$PLCPID" -echo "telescopeSimulator finished" -echo "" - -echo "Waiting for cameraNIRSimulator to finish" -wait "$CAMERANIRPID" -echo "cameraNIRSimulator finished" -echo "" - -echo "Waiting for cameraVISSimulator to finish" -wait "$CAMERAVISPID" -echo "cameraVISSimulator finished" -echo "" - -echo "Killing all processes" - -kill_sim \ No newline at end of file diff --git a/simulators/scripts/start_simulator_worker.sh b/simulators/scripts/start_simulator_worker.sh deleted file mode 100755 index a356d40..0000000 --- a/simulators/scripts/start_simulator_worker.sh +++ /dev/null @@ -1,18 +0,0 @@ -# Kill and re-launch simulator workers - -source ../private/venv_py3_pyros/bin/activate - -./scripts/kill_celery_workers.sh - -celery worker -A pyros -Q monitoring_q -n pyros@monitoring -c 1 & -celery worker -A pyros -Q alert_listener_q -n pyros@alert_listener -c 1 & -celery worker -A pyros -Q majordome_q -n pyros@majordome -c 1 & - -celery worker -A pyros -Q scheduling_q --purge -n pyros@scheduling -c 1 & -celery worker -A pyros -Q execute_sequence_q --purge -n pyros@execute_sequence -c 1 & -celery worker -A pyros -Q execute_plan_vis_q --purge -n pyros@execute_plan_vis -c 1 & -celery worker -A pyros -Q execute_plan_nir_q --purge -n pyros@execute_plan_nir -c 1 & -celery worker -A pyros -Q create_calibrations_q --purge -n pyros@create_calibrations -c 1 & -celery worker -A pyros -Q analysis_q --purge -n pyros@analysis -c 5 & -celery worker -A pyros -Q system_status_q --purge -n pyros@system_status -c 1 & -celery worker -A pyros -Q change_obs_conditions_q --purge -n pyros@change_obs_conditions -c 1 & diff --git a/simulators/user/userSimulator.py b/simulators/user/userSimulator.py index c4e23b3..f0087c5 100644 --- a/simulators/user/userSimulator.py +++ b/simulators/user/userSimulator.py @@ -10,7 +10,7 @@ class UserSimulator(): conf_file = "../config/conf.json" conf_path = "../config/" request_path = "../resources/" - login = "http://localhost:8000/user_manager/login" #  + login = "http://localhost:8000/user_manager/login" url = "http://localhost:8000/routine_manager/import_request" get_url = "http://localhost:8000/routine_manager/" sims = [] @@ -51,16 +51,22 @@ class UserSimulator(): def authenticate(self): self.client = requests.session() - self.client.get("http://localhost:8000/user_manager/login") - csrftoken = self.client.cookies['csrftoken'] - self.logindata = dict(email="pyros", password="DjangoPyros", csrfmiddlewaretoken=csrftoken) - resp = self.client.post(self.login, data=self.logindata, headers=dict(Referer=self.login)) - self.userPrint("Connection status : %d"%(resp.status_code)) + try: + self.client.get("http://localhost:8000/user_manager/login") + csrftoken = self.client.cookies['csrftoken'] + self.logindata = dict(email="pyros", password="DjangoPyros", csrfmiddlewaretoken=csrftoken) + resp = self.client.post(self.login, data=self.logindata, headers=dict(Referer=self.login)) + self.userPrint("Connection status : %d"%(resp.status_code)) + except Exception as e: + print(e, file=sys.stderr) + return 1 + return 0 def run(self): i = 0 self.parse() - self.authenticate() + if self.authenticate(): + return 1 self.userPrint("The simulator will end in %d seconds"%(int(self.ended))) while (i < self.ended): for dic in self.sims: @@ -72,4 +78,4 @@ class UserSimulator(): if (__name__ == "__main__"): sim = UserSimulator(sys.argv) - sim.run() + sys.exit(sim.run()) diff --git a/src/scripts/celery_test.sh b/src/scripts/celery_test.sh deleted file mode 100755 index 890c1e5..0000000 --- a/src/scripts/celery_test.sh +++ /dev/null @@ -1,64 +0,0 @@ -# -## To be used for tests using celery (asynchronous) -## BEWARE : This script launches tests on the production database -# - -# -## Usage : celery_test.sh app.tests[.ClassTest[.test_name]] -# - -USAGE_MSG="Usage : celery_test.sh app.tests[.ClassTest[.test_name]]" -CELERY_WORKER_OFF="You must run scripts/start_celery_workers.sh before running this test" - -if [ "$#" -eq 0 ]; then - echo $USAGE_MSG - exit -fi - -# Checks if the workers are running. Useless now because we run them from the script - -# ALERT_WORKER=`ps aux | grep "celery worker" | grep alert_listener` -# if [ "${#ALERT_WORKER}" -eq 0 ]; then -# echo $CELERY_WORKER_OFF -# exit -# fi - -SCRIPT_PATH=$0 -SCRIPT_PATH=${SCRIPT_PATH%/*} - -cd $SCRIPT_PATH/.. - -# Sets the variable CELERY_TEST in settings.py in order to use to good test DB -sed -i -e "s/CELERY_TEST = False/CELERY_TEST = True/g" pyros/settings.py - -# Deletes the DB, then re-create it with basic fixtures -rm -f testdb.sqlite3 -python manage.py migrate -python manage.py loaddata misc/fixtures/initial_fixture.json - -# Starts the simulators -cd ../simulators/scripts 2>&1 /dev/null -./run_all.sh -cd - - -# Start celery -./scripts/start_celery_single_worker.sh 2>&1 /dev/null - -# Run the tests -python manage.py test "$@" --keepdb --nomigrations - -# Reset the variable in settings.py -sed -i -e "s/CELERY_TEST = True/CELERY_TEST = False/g" pyros/settings.py - -# Delete the test DB -rm -f testdb.sqlite3 - -cd scripts/ - -# Stops celery workers -./kill_celery_workers.sh - -# Stops simulators -cd ../../simulators/scripts/ -./kill_all.sh -cd - diff --git a/src/scripts/delete_all_requests.sh b/src/scripts/delete_all_requests.sh deleted file mode 100755 index 007d7d5..0000000 --- a/src/scripts/delete_all_requests.sh +++ /dev/null @@ -1,19 +0,0 @@ -# This mini-script deletes all your alerts, requests, alerts, albums, plans, shs and schedules in db -# You need to have a root user with no password on your DB - -SCRIPT_PATH=$0 -SCRIPT_PATH=${SCRIPT_PATH%/*} - -cd $SCRIPT_PATH/.. - -MYSQL=`grep "MYSQL = " pyros/settings.py` -MYSQL=${MYSQL##* } -MYSQL=${MYSQL%?} - -REQUEST="delete from plan ; delete from album ; delete from sequence ; delete from alert ; delete from request ; delete from schedule_has_sequences ; delete from schedule ; delete from log" - -if [ "$MYSQL" == "True" ]; then - mysql -u root -e "$REQUEST" pyros -else - sqlite3 db.sqlite3 "$REQUEST" -fi diff --git a/src/scripts/kill_celery_workers.bat b/src/scripts/kill_celery_workers.bat deleted file mode 100644 index 5d9a89d..0000000 --- a/src/scripts/kill_celery_workers.bat +++ /dev/null @@ -1,3 +0,0 @@ -REM deletes all running celery workers - -taskkill /f /im celery.exe \ No newline at end of file diff --git a/src/scripts/kill_celery_workers.sh b/src/scripts/kill_celery_workers.sh deleted file mode 100755 index aadeacd..0000000 --- a/src/scripts/kill_celery_workers.sh +++ /dev/null @@ -1,3 +0,0 @@ -# Mini-script to kill all the celery-workers in background - -ps aux | grep "celery worker" | awk '{print $2}' | xargs kill -9 diff --git a/src/scripts/kill_simulation.sh b/src/scripts/kill_simulation.sh deleted file mode 100755 index e89ea5d..0000000 --- a/src/scripts/kill_simulation.sh +++ /dev/null @@ -1,25 +0,0 @@ -cd .. - -#Reseting -sed -i -e "s/CELERY_TEST = True/CELERY_TEST = False/g" pyros/settings.py -# sed -i -e "s/.*SIMULATOR_VALUE = .*/SIMULATOR_VALUE = None/g" userSimulator/tasks.py - -#Kill web server -fuser -k 8000/tcp - -# Delete the test DB -rm -f testdb.sqlite3 - -ps aux | grep "userSimulator.py" | awk '{ print $2 }' | xargs kill -ps aux | grep "alertSimulator.py" | awk '{ print $2 }' | xargs kill -ps aux | grep "plcSimulator.py" | awk '{ print $2 }' | xargs kill -ps aux | grep "telescopeSimulator.py" | awk '{ print $2 }' | xargs kill -ps aux | grep "cameraNIRSimulator.py" | awk '{ print $2 }' | xargs kill -ps aux | grep "cameraVISSimulator.py" | awk '{ print $2 }' | xargs kill - -cd scripts/ - -# Stops celery workers -./kill_celery_workers.sh - -echo "--------> Simulation ended <----------" diff --git a/src/scripts/start_celery_single_worker.sh b/src/scripts/start_celery_single_worker.sh deleted file mode 100755 index b0f7f3c..0000000 --- a/src/scripts/start_celery_single_worker.sh +++ /dev/null @@ -1,21 +0,0 @@ -# Kills and re-launches the celery workers - -SCRIPT_PATH=$0 -SCRIPT_PATH=${SCRIPT_PATH%/*} - -cd $SCRIPT_PATH/.. - -source ../private/venv_py3_pyros/bin/activate - -./scripts/kill_celery_workers.sh -celery worker -A pyros -Q $1_q -n pyros@$1 -c 1 & -# celery worker -A pyros -Q monitoring_q -n pyros@monitoring -c 1 & -# celery worker -A pyros -Q majordome_q -n pyros@majordome -c 1 & -# celery worker -A pyros -Q scheduling_q --purge -n pyros@scheduling -c 1 & -# celery worker -A pyros -Q execute_sequence_q --purge -n pyros@execute_sequence -c 1 & -# celery worker -A pyros -Q execute_plan_vis_q --purge -n pyros@execute_plan_vis -c 1 & -# celery worker -A pyros -Q execute_plan_nir_q --purge -n pyros@execute_plan_nir -c 1 & -# celery worker -A pyros -Q create_calibrations_q --purge -n pyros@create_calibrations -c 1 & -# celery worker -A pyros -Q analysis_q --purge -n pyros@analysis -c 5 & -# celery worker -A pyros -Q system_status_q --purge -n pyros@system_status -c 1 & -# celery worker -A pyros -Q change_obs_conditions_q --purge -n pyros@change_obs_conditions -c 1 & diff --git a/src/scripts/start_celery_workers.bat b/src/scripts/start_celery_workers.bat deleted file mode 100644 index e08cee0..0000000 --- a/src/scripts/start_celery_workers.bat +++ /dev/null @@ -1,18 +0,0 @@ -REM Kills and relaunches celery workers - -REM Moves into the PYROS\src directory - -cd %~dp0% -cd .. - -call scripts\kill_celery_workers.bat - -START /B "" celery worker -A pyros -Q alert_listener_q -n pyros@alert_listener -c 1 -START /B "" celery worker -A pyros -Q scheduling_q --purge -n pyros@scheduling -c 1 -START /B "" celery worker -A pyros -Q execute_sequence_q --purge -n pyros@execute_sequence -c 1 -START /B "" celery worker -A pyros -Q execute_plan_vis_q --purge -n pyros@execute_plan_vis -c 1 -START /B "" celery worker -A pyros -Q execute_plan_nir_q --purge -n pyros@execute_plan_nir -c 1 -START /B "" celery worker -A pyros -Q analysis_q --purge -n pyros@analysis -c 5 -START /B "" celery worker -A pyros -Q system_status_q --purge -n pyros@system_status -c 1 -START /B "" celery worker -A pyros -Q change_obs_conditions_q --purge -n pyros@change_obs_conditions -c 1 -START /B "" celery worker -A pyros -Q monitoring_q -n pyros@monitoring -c 1 diff --git a/src/scripts/start_celery_workers.sh b/src/scripts/start_celery_workers.sh deleted file mode 100755 index 4f5f06a..0000000 --- a/src/scripts/start_celery_workers.sh +++ /dev/null @@ -1,23 +0,0 @@ -# Kills and re-launches the celery workers - -SCRIPT_PATH=$0 -SCRIPT_PATH=${SCRIPT_PATH%/*} - -cd $SCRIPT_PATH/.. - -source ../private/venv_py3_pyros/bin/activate - -./scripts/kill_celery_workers.sh -celery worker -A pyros -Q alert_listener_q -n pyros@alert_listener -c 1 & -celery worker -A pyros -Q monitoring_q -n pyros@monitoring -c 1 & -celery worker -A pyros -Q majordome_q -n pyros@majordome -c 1 & - -celery worker -A pyros -Q scheduling_q --purge -n pyros@scheduling -c 1 & -celery worker -A pyros -Q execute_sequence_q --purge -n pyros@execute_sequence -c 1 & -celery worker -A pyros -Q execute_plan_vis_q --purge -n pyros@execute_plan_vis -c 1 & -celery worker -A pyros -Q execute_plan_nir_q --purge -n pyros@execute_plan_nir -c 1 & -celery worker -A pyros -Q create_calibrations_q --purge -n pyros@create_calibrations -c 1 & -celery worker -A pyros -Q analysis_q --purge -n pyros@analysis -c 5 & -celery worker -A pyros -Q system_status_q --purge -n pyros@system_status -c 1 & -celery worker -A pyros -Q change_obs_conditions_q --purge -n pyros@change_obs_conditions -c 1 & - diff --git a/src/scripts/test_all_usual.sh b/src/scripts/test_all_usual.sh deleted file mode 100755 index 2b46698..0000000 --- a/src/scripts/test_all_usual.sh +++ /dev/null @@ -1,11 +0,0 @@ -# -## Launches all the tests not needing comet or celery -# - - -SCRIPT_PATH=$0 -SCRIPT_PATH=${SCRIPT_PATH%/*} - -cd $SCRIPT_PATH/.. - -../private/venv_py3_pyros/bin/python3 manage.py test common scheduler routine_manager user_manager alert_manager.tests.TestStrategyChange -- libgit2 0.21.2