import sys import os import subprocess import platform import fileinput import argparse import time import signal DEBUG = False class Utils: system = platform.system() columns = 100 row = 1000 disp = True def __init__(self): if (platform.system() != 'Windows'): try: rows, columns = os.popen('stty size', 'r').read().split() self.columns = int(columns) except: self.columns = 100 if DEBUG: print("Could not get terminal size") def printFullTerm(self, color, string): value = int(self.columns / 2 - len(string) / 2) self.printColor(color, "-" * value, eol='') self.printColor(color, string, eol='') value += len(string) self.printColor(color, "-" * (self.columns - value)) return 0 def changeDirectory(self, path): if DEBUG: print("Moving to : " + path) os.chdir(path) if DEBUG: print("Current directory : " + str(os.getcwd())) return 0 def replacePatternInFile(self, pattern, replace, file_path): try: with fileinput.FileInput(file_path, inplace=True, backup='.bak') as file: for line in file: print(line.replace(pattern, replace), end='') except: return 1 return 0 def printColor(self, color, message, file=sys.stdout, eol=os.linesep, forced=False): if (self.disp == False and forced == False): return 0 if (self.system == 'Windows'): print(message, file=file, end=eol) else: print(color + message + Colors.ENDC, file=file, end=eol) return 0 def askQuestion(self, message, default = ""): self.printColor(Colors.BLUE, message, forced=True) self.printColor(Colors.BOLD, "Answer (default="+default+"): ", eol='', forced=True) sys.stdout.flush() ret = sys.stdin.readline().replace('\n', '').replace('\r', '') if ret == "": return default return ret def sleep(self, t): time.sleep(t) return 0 ''' Manager class : manager of your project ''' class AManager(Utils): path = os.path.realpath(__file__) path_dir = os.getcwd() path_dir_file = os.path.dirname(os.path.realpath(__file__)) python_path = sys.executable python_version = sys.version_info bin_dir = "" celery = "celery" venv_pip = "pip" venv_bin = "python" wait = True current_command = "" config = None commandMatcher = {} commandDescription = {} commands = [] errors = {} executed = {} subproc = [] def __init__(self, param): super(AManager, self).__init__() self.wait = param.getWait() self.commands = param.getCommandList() self.disp = param.getPrint() config = param.getConfig() self.config = config signal.signal(signal.SIGINT, self.signal_handler) self.changeDirectory(self.path_dir_file) if self.system == 'Windows': self.bin_dir = "Scripts" self.bin_name = "python.exe" self.pip_name = "pip.exe" self.celery = "celery.exe" else: self.bin_dir = "bin" self.bin_name = "python" self.pip_name = "pip" self.celery = "celery" self.venv_pip = self.path_dir_file + os.sep + config["path"] + os.sep + config["env"] + os.sep + self.bin_dir + os.sep + self.pip_name self.venv_bin = self.path_dir_file + os.sep + config["path"] + os.sep + config["env"] + os.sep + self.bin_dir + os.sep + self.bin_name self.venv_cel = self.path_dir_file + os.sep + config["path"] + os.sep + config["env"] + os.sep + self.bin_dir + os.sep + self.celery def help(self): print("This function must be implemented") raise(NotImplementedError("Function not implemented")) def signal_handler(self, signal, frame): self.printFullTerm(Colors.WARNING, "Ctrl-c catched") for p in self.subproc: proc, name = p self.printColor(Colors.BLUE, "Killing process " + str(name)) proc.kill() self.printFullTerm(Colors.WARNING, "Exiting") sys.exit(0) def addExecuted(self, src, message): if (src in self.executed): self.executed[src].append(str(message)) else: self.executed[src] = [str(message)] return 0 def addError(self, src, message): if (src in self.errors): self.errors[src].append(str(message)) else: self.errors[src] = [str(message)] return 0 def execProcess(self, command): self.printFullTerm(Colors.BLUE, "Executing command [" + command + "]") process = subprocess.Popen(command, shell=True) process.wait() if process.returncode == 0: self.printFullTerm(Colors.GREEN, "Process executed successfully") self.addExecuted(self.current_command, command) else: self.printFullTerm(Colors.WARNING, "Process execution failed") self.addError(self.current_command, command) return process.returncode def execProcessSilent(self, command): process = subprocess.Popen(command, shell=True) process.wait() return process.returncode def execProcessFromVenv(self, command): args = command.split() self.printFullTerm(Colors.BLUE, "Executing command from venv [" + str(' '.join(args[1:])) + "]") process = subprocess.Popen(args) process.wait() if process.returncode == 0: self.printFullTerm(Colors.GREEN, "Process executed successfully") self.addExecuted(self.current_command, str(' '.join(args[1:]))) else: self.printFullTerm(Colors.WARNING, "Process execution failed") self.addError(self.current_command, str(' '.join(args[1:]))) return process.returncode def execProcessAsync(self, command): self.printFullTerm(Colors.BLUE, "Executing command [" + command + "]") p = subprocess.Popen(command, shell=True) self.subproc.append((p, command)) self.printFullTerm(Colors.GREEN, "Process launched successfully") self.addExecuted(self.current_command, command) return p def execProcessFromVenvAsync(self, command: str): args = command.split() self.printFullTerm(Colors.BLUE, "Executing command from venv [" + str(' '.join(args[1:])) + "]") p = subprocess.Popen(args) self.subproc.append((p, ' '.join(args[1:]))) self.printFullTerm(Colors.GREEN, "Process launched successfully") self.addExecuted(self.current_command, str(' '.join(args[1:]))) return p def waitProcesses(self): if (self.wait): for p in self.subproc: proc, name = p proc.wait() return 0 def end(self): count = 0 self.waitProcesses() self.printFullTerm(Colors.WARNING, "Summary") self.printColor(Colors.GREEN, "Success : ") for command, valid in self.executed.items(): if not valid: self.printColor(Colors.BLUE, "\t- Command : " + command + " success !") else: self.printColor(Colors.WARNING, "\t- In commmand : " + command) for exe in valid: self.printColor(Colors.GREEN, "\t\t - Command : " + exe + " success !") self.printColor(Colors.FAIL, "Errors : ") if not self.errors: self.printColor(Colors.GREEN, "\tNone") for command, items in self.errors.items(): count += 1 if (not items): self.printColor(Colors.FAIL, "Command : " + command + " failed !") else: self.printColor(Colors.WARNING, "\t- In commmand : " + command) for exe in items: self.printColor(Colors.FAIL, "\t\t - Command : " + exe) return count def exec(self): if (not self.commands): self.commandMatcher["help"]() return 0 for command in self.commands: self.current_command = command if command in self.commandMatcher: self.commandMatcher[command]() else: self.addError(str(command), "invalid command") return self.end() def logError(self, message): self.printColor(Colors.FAIL, "Pyros : An error occurred [" + message + "]", file=sys.stderr) return 0 class Config: __parser = argparse.ArgumentParser("Installer parser") __content = { "path": "private", "env": "venv_py3_pyros" } __wait = True __print = True usage = "" __command_list = [] def __init__(self): self.__parser.add_argument("command", nargs='?', default="help", help="The command you want to execute (default=help)") self.__parser.add_argument("--env", help="Your environment directory name default=venv") self.__parser.add_argument("--path", help="Path to the virtual env (from the source file directory) (default=private)") self.__parser.add_argument("--nowait", action='store_true', help="Don't wait the end of a program") self.__parser.add_argument("--noprint", action='store_true', help="Won't print") self.usage = self.__parser.format_usage() def parse(self): try: res = self.__parser.parse_args(), self.__parser.format_usage() return (res) except SystemExit as e: # print(e, file=sys.stderr) sys.exit(1) def parseConf(self): res, usage = self.parse() try: if (res.env): self.__content["env"] = res.env if (res.path): self.__content["path"] = res.path if (res.nowait): self.__wait = False if (res.noprint): self.__print = False self.__command_list.append(res.command) return 0 except Exception as e: print(e, file=sys.stderr) return 1 def getPath(self): return self.__content["path"] def getEnv(self): return self.__content["env"] def getWait(self): return self.__wait def getPrint(self): return self.__print def setPrint(self, value): self.__print = value return 0 def setWait(self, value): self.__wait = value return 0 def getCommandList(self): return self.__command_list def printUsage(self): print(self.usage, file=sys.stderr) def addConf(self, key, value): if isinstance(key, str) and isinstance(value, str): self.__content[key] = value return 0 return 1 def setPath(self, path): if (os.path.isdir(path)): if (path == ""): path = "." self.__content["path"] = path return 0 return 1 def setEnvName(self, name): self.__content["env"] = name return 0 def getConfig(self): return self.__content ''' Color class ''' class Colors: HEADER = '\033[95m' BLUE = '\033[94m' GREEN = '\033[92m' WARNING = '\033[93m' FAIL = '\033[91m' ENDC = '\033[0m' BOLD = '\033[1m' UNDERLINE = '\033[4m' ''' Pyros class ''' class Pyros(AManager): help_message = "python neo.py" init_fixture = "initial_fixture.json" def install(self): if (self.system == "Windows"): self.execProcess("python install/install.py install") else: self.execProcess("python3 install/install.py install") return 0 def update(self): if (self.system == "Windows"): self.execProcess("python install/install.py update") else: self.execProcess("python3 install/install.py update") return 0 def server(self): self.changeDirectory("src") self.execProcessFromVenvAsync(self.venv_bin + " manage.py runserver") self.changeDirectory("..") return 0 def clean(self): return self.clean_logs() def clean_logs(self): return self.execProcess("rm logs/*.log") def test(self): self.changeDirectory("src") self.execProcessFromVenvAsync(self.venv_bin + " manage.py test") self.changeDirectory("..") return 0 def migrate(self): self.changeDirectory("src") self.execProcessFromVenv(self.venv_bin + " manage.py migrate") self.changeDirectory("..") return 0 def makemigrations(self): self.changeDirectory("src") self.execProcessFromVenv(self.venv_bin + " manage.py makemigrations") self.changeDirectory("..") return 0 def help(self): count = 0 self.printFullTerm(Colors.WARNING, "Help Message") for command, message in self.commandDescription.items(): count += 1 if (self.columns > 100): self.printColor(Colors.BLUE, "\t"+str(count)+(' ' if count < 10 else '')+": " + command + ": ", eol='') else: self.printColor(Colors.BLUE, "-> " + command + ": ", eol='') self.printColor(Colors.GREEN, message) return 0 def updatedb(self): self.makemigrations() self.migrate() return 0 def unittest(self): self.changeDirectory("src") self.execProcessFromVenv(self.venv_bin + " manage.py test common scheduler routine_manager user_manager alert_manager.tests.TestStrategyChange") self.changeDirectory("..") return 0 def test_all(self): self.unittest() # TODO Fix this # self.changeDirectory("src") # self.replacePatternInFile("CELERY_TEST = False", "CELERY_TEST = True", "pyros/settings.py") # self.execProcess("rm -f testdb.sqlite3") # self.changeDirectory("..") # # self.migrate() # self.loaddata() # self.sims_launch() # self.singleWorker("alert_manager") # self.changeDirectory("src") # self.execProcessFromVenv(self.venv_bin + " manage.py test alert_manager.tests.AlertListenerTestsCelery --keepdb --nomigrations") # self.replacePatternInFile("CELERY_TEST = True", "CELERY_TEST = False", "pyros/settings.py") # self.execProcess("rm -f testdb.sqlite3") # self.changeDirectory("..") # self.kill_simulation() return 0 def loaddata(self): self.changeDirectory("src") self.execProcessFromVenv(self.venv_bin + " manage.py loaddata misc/fixtures/" + self.init_fixture) self.changeDirectory("..") return 0 def celery_on(self): self.changeDirectory("src") self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q alert_listener_q -n pyros@alert_listener -c 1") self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q monitoring_q -n pyros@monitoring -c 1") self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q majordome_q -n pyros@majordome -c 1") self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q scheduling_q --purge -n pyros@scheduling -c 1") self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q execute_sequence_q --purge -n pyros@execute_sequence -c 1") self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q execute_plan_vis_q --purge -n pyros@execute_plan_vis -c 1") self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q execute_plan_vis_q --purge -n pyros@execute_plan_nir -c 1") self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q create_calibrations_q --purge -n pyros@create_calibrations -c 1") self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q analysis_q --purge -n pyros@analysis -c 1") self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q system_status_q --purge -n pyros@system_status -c 1") self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q change_obs_conditions_q --purge -n pyros@change_obs_conditions -c 1") self.changeDirectory("..") return 0 def start(self): self.stop() self.celery_on() return 0 def stop(self): if (self.system == "Windows"): self.execProcessAsync("taskkill /f /im celery.exe") self.execProcessAsync("taskkill /f /im python.exe") else: self.execProcessAsync("ps aux | grep \"celery worker\" | awk '{print $2}' | xargs kill -9") return 0 def init_database(self): self.makemigrations() self.migrate() self.loaddata() return 0 def simulator_development(self): self.changeDirectory("src") self.replacePatternInFile("CELERY_TEST = False", "CELERY_TEST = True", "pyros/settings.py") self.execProcess("rm -f testdb.sqlite3") self.changeDirectory("..") self.migrate() self.loaddata() self.server() self.sleep(2) self.printFullTerm(Colors.WARNING, "SUMMARY") self.printColor(Colors.GREEN, "The simulator has been successfully initialised") self.printColor(Colors.GREEN, "The simulator run on a temp database : src/testdb.sqlite3") self.printColor(Colors.GREEN, "The simulation will be ended by the task 'simulator herself'") self.printColor(Colors.GREEN, "If you want to shutdown the simulation, please run :") self.printColor(Colors.GREEN, "CTRL-C or ./pyrosrun.sh kill_simulation") self.printColor(Colors.GREEN, "If the simulation isn't correctly killed, please switch the variable") self.printColor(Colors.GREEN, "CELERY_TEST in src/pyros/settings.py to false") self.printFullTerm(Colors.WARNING, "SUMMARY") self.changeDirectory("simulators/config") self.printColor(Colors.BOLD, "Existing simulations : ", eol='') sys.stdout.flush() self.execProcessSilent("ls conf*.json") self.changeDirectory("..") conf = self.askQuestion("Which simulation do you want to use", default="conf.json") self.changeDirectory("..") self.singleWorker("scheduling") self.singleWorker("majordome") self.sleep(3) procs = [] self.changeDirectory("simulators") self.changeDirectory("user") procs.append(self.execProcessFromVenvAsync(self.venv_bin + " userSimulator.py " + conf)) self.changeDirectory("..") for p in procs: p.wait() self.changeDirectory("..") self.kill_simulation() return 0 def simulator(self): self.changeDirectory("src") self.replacePatternInFile("CELERY_TEST = False", "CELERY_TEST = True", "pyros/settings.py") self.execProcess("rm -f testdb.sqlite3") self.changeDirectory("..") self.migrate() self.loaddata() self.server() self.sleep(2) self.printFullTerm(Colors.WARNING, "SUMMARY") self.printColor(Colors.GREEN, "The simulator has been successfully initialised") self.printColor(Colors.GREEN, "The simulator run on a temp database : src/testdb.sqlite3") self.printColor(Colors.GREEN, "The simulation will be ended by the task 'simulator herself'") self.printColor(Colors.GREEN, "If you want to shutdown the simulation, please run :") self.printColor(Colors.GREEN, "CTRL-C or ./pyrosrun.sh kill_simulation") self.printColor(Colors.GREEN, "If the simulation isn't correctly killed, please switch the variable") self.printColor(Colors.GREEN, "CELERY_TEST in src/pyros/settings.py to false") self.printFullTerm(Colors.WARNING, "SUMMARY") self.changeDirectory("simulators/config") self.printColor(Colors.BOLD, "Existing simulations : ", eol='') sys.stdout.flush() self.execProcessSilent("ls conf*.json") self.changeDirectory("..") conf = self.askQuestion("Which simulation do you want to use", default="conf.json") self.changeDirectory("..") self.celery_on() self.sleep(3) self.sims_launch(conf) return 0 def kill_server(self): self.execProcessAsync("fuser -k 8000/tcp") return 0 def kill_simulation(self): self.changeDirectory("src") self.replacePatternInFile("CELERY_TEST = True", "CELERY_TEST = False", "pyros/settings.py") if (self.system == "Windows"): self.execProcessAsync("taskkill /f /im python.exe") self.execProcessAsync("rm -f testdb.sqlite3") self.changeDirectory("..") return 0 self.execProcessAsync("fuser -k 8000/tcp") self.execProcessAsync("rm -f testdb.sqlite3") self.execProcessAsync("ps aux | grep \" domeSimulator.py\" | awk '{ print $2 }' | xargs kill") self.execProcessAsync("ps aux | grep \" userSimulator.py\" | awk '{ print $2 }' | xargs kill") self.execProcessAsync("ps aux | grep \" alertSimulator.py\" | awk '{ print $2 }' | xargs kill") self.execProcessAsync("ps aux | grep \" plcSimulator.py\" | awk '{ print $2 }' | xargs kill") self.execProcessAsync("ps aux | grep \" telescopeSimulator.py\" | awk '{ print $2 }' | xargs kill") self.execProcessAsync("ps aux | grep \" cameraNIRSimulator.py\" | awk '{ print $2 }' | xargs kill") self.execProcessAsync("ps aux | grep \" cameraVISSimulator.py\" | awk '{ print $2 }' | xargs kill") self.changeDirectory("..") self.stop() self.printFullTerm(Colors.GREEN, "simulation ended") return 0 def sims_launch(self, conf=""): procs = [] self.changeDirectory("simulators/config") if (conf == ""): self.printColor(Colors.BOLD, "Existing simulations : ", eol='') sys.stdout.flush() self.execProcessSilent("ls conf*.json") conf = self.askQuestion("Which simulation do you want to use ?", default="conf.json") if not os.path.isfile(conf): self.printColor(Colors.FAIL, "The simulation file " + conf + " does not exist") return 1 self.changeDirectory("..") self.changeDirectory("dome") procs.append(self.execProcessFromVenvAsync(self.venv_bin + " domeSimulator.py " + conf)) self.changeDirectory("..") self.changeDirectory("user") procs.append(self.execProcessFromVenvAsync(self.venv_bin + " userSimulator.py " + conf)) self.changeDirectory("..") self.changeDirectory("alert") procs.append(self.execProcessFromVenvAsync(self.venv_bin + " alertSimulator.py " + conf)) self.changeDirectory("..") self.changeDirectory("plc") procs.append(self.execProcessFromVenvAsync(self.venv_bin + " plcSimulator.py " + conf)) self.changeDirectory("..") self.changeDirectory("camera") procs.append(self.execProcessFromVenvAsync(self.venv_bin + " cameraVISSimulator.py " + conf)) procs.append(self.execProcessFromVenvAsync(self.venv_bin + " cameraNIRSimulator.py " + conf)) self.changeDirectory("..") self.changeDirectory("telescope") procs.append(self.execProcessFromVenvAsync(self.venv_bin + " telescopeSimulator.py " + conf)) self.changeDirectory("..") self.changeDirectory("..") for p in procs: p.wait() self.kill_simulation() return 0 def singleWorker(self, worker): self.changeDirectory("src") self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q "+ worker +"_q -n pyros@"+worker+" -c 1") self.changeDirectory("..") return 0 def __init__(self, argv): super(Pyros, self).__init__(argv) self.commandMatcher = { "install": self.install, "update": self.update, "server": self.server, "clean": self.clean, "simulator_scheduler": self.simulator_development, "clean_logs": self.clean_logs, "test": self.test, "migrate": self.migrate, "makemigrations": self.makemigrations, "updatedb": self.updatedb, "unittest": self.unittest, "test_all": self.test_all, "celery_on": self.celery_on, "init_database": self.init_database, "kill_server": self.kill_server, "loaddata": self.loaddata, "start": self.start, "stop": self.stop, "simulator": self.simulator, "kill_simulation": self.kill_simulation, "sims_launch": self.sims_launch, "help": self.help, } self.commandDescription = { "install": "Launch the server installation", "update": "Update the server", "server": "Launch the web server", "loaddata": "Load the initial fixture in database", "clean": "clean the repository", "clean_logs": "clean the log directory", "test": "launch the server tests", "migrate": "execute migrations", "makemigrations": "create new migrations", "help": "Help message", "updatedb": "Update the database", "kill_server": "Kill the web server on port 8000", "init_database": "Create a standard context for pyros in db", "unittest": "Runs the tests thaht don't need celery", "test_all": "Run all the existing tests (this command needs to be updated when tests are added in the project", "celery_on": "Starts celery workers", "start": "Stop the celery workers then the web server", "stop": "stops the celery workers", "simulator": "Launch a simulation", "simulator_scheduler": "Simulation for the scheduler only", "kill_simulation": "kill the simulators / celery workers / web server", "sims_launch": "Launch only the simulators", } if __name__ == "__main__": conf = Config() if conf.parseConf(): sys.exit(1) pyros = Pyros(conf) sys.exit(pyros.exec())