#!/usr/bin/env python3 import argparse import fileinput import glob import os import platform import shutil import signal import subprocess import sys import time import re """ ***************************************************************** ******************** GENERAL CONSTANTS ************************** ***************************************************************** """ #DEBUG = False UPDATE = False PYROS_DJANGO_BASE_DIR = "src/core/pyros_django" INIT_FIXTURE = "initial_fixture.json" _previous_dir = None AGENTS = { #"agentX" : "agent", "agent" : "Agent", "agentX" : "AgentX", "agentA" : "AgentA", "agentB" : "AgentB", "agentM" : "AgentM", #"agentDevice" : "AgentDevice", #"agentDeviceTelescopeGemini" : "AgentDeviceTelescopeGemini", "agentDeviceGemini" : "AgentDeviceGemini", "agentDeviceSBIG" : "AgentDeviceSBIG", "agentTelescopeRequester" : "AgentTelescopeRequester", "agentMultiRequester" : "AgentMultiRequester", "webserver" : "webserver", "monitoring" : "monitoring", "majordome" : "majordome", "scheduler" : "scheduler", "alert_manager" : "alert_manager" } #AGENTS = ["agentX", "webserver", "monitoring", "majordome", "scheduler", "alert_manager"] #AGENTS = ["all", "webserver", "monitoring", "majordome", "scheduler", "alert"] #COMMANDS = {"install": [], "start": AGENTS, "stop": AGENTS} REQUIREMENTS = 'requirements.txt' b_in_dir = "bin" PYTHON = "python3" # should also be ok from venv: #PYTHON = "python" IS_WINDOWS = platform.system() == "Windows" if IS_WINDOWS: REQUIREMENTS = 'requirements_win.txt' b_in_dir = "Scripts" PYTHON = "python.exe" #VENV_ROOT = "private" VENV_ROOT = "venv" VENV = "venv_py3_pyros" my_abs_path = os.path.dirname(os.path.realpath(__file__)) VENV_BIN = ( my_abs_path + os.sep + VENV_ROOT + os.sep + VENV + os.sep + b_in_dir + os.sep ) VENV_PYTHON = VENV_BIN + PYTHON VENV_PIP = VENV_BIN + 'pip' class Colors: HEADER = "\033[95m" BLUE = "\033[94m" GREEN = "\033[92m" WARNING = "\033[93m" FAIL = "\033[91m" ENDC = "\033[0m" BOLD = "\033[1m" UNDERLINE = "\033[4m" ERROR = '\033[91m' END = '\033[0m' LOG_BLUE = '\033[94m' if IS_WINDOWS: ERROR = '' END = '' LOG_BLUE = '' ''' VARIABLES FOR INSTALL ''' # By default, install the virtual environment AND the database INSTALL_VENV = True INSTALL_DB = True SQL_DATABASE = "pyros" # Database automatically created by Django for tests (and automatically deleted after tests) SQL_DATABASE_TEST = "test_pyros" # Specific database that we need(ed?) for some simulations SQL_DATABASE_SIMU = "pyros_test" SQL_USER = "pyros" SQL_PSWD = "DjangoPyros" MYSQL_EXE_PATH = "" try : # WITH_DOCKER is an environment varialbe from our Docker image WITH_DOCKER=os.environ['WITH_DOCKER'] except KeyError: WITH_DOCKER=False if type(WITH_DOCKER) is str and re.match("^y$|^Y$|^yes$|^Yes$",WITH_DOCKER.rstrip()) != None : WITH_DOCKER = True else : WITH_DOCKER = False ENV_PATH = "docker/.env" ENV_SAMPLE_PATH = "docker/.env-sample" is_environment_variables_not_defined = os.environ.get("MYSQL_ROOT_PASSWORD") == None if(is_environment_variables_not_defined): try: with open(ENV_PATH,"r") as env_file: print("Reading env file") for line in env_file: if(line.startswith("#") and not env_sample_line.strip()): continue else: key,value = line.split("=") # setting variables as environment variables os.environ[key] = value except: print(f".env not found at {ENV_PATH}, creating a file at this path from the .env-sample file stored at {ENV_SAMPLE_PATH}\n \ values from .env-sample will be used as environment variables") with open(ENV_SAMPLE_PATH,'r') as env_sample_file: with open(ENV_PATH,"w") as env_file: for env_sample_line in env_sample_file: if(env_sample_line.startswith("#") or not env_sample_line.strip()): continue key,value = env_sample_line.split("=") os.environ[key] = value env_file.write(env_sample_line) is_environment_variables_not_defined = os.environ.get("MYSQL_ROOT_PASSWORD") == None END_OF_LINE = '\n\n' ##VENV_BIN = '/bin/' # -------------------------------------------- # --- Modified values for Windows # -------------------------------------------- if (platform.system() == "Windows"): END_OF_LINE = "\r\n\r\n" ##VENV_BIN = '\\Scripts\\' #MYSQL_EXE_PATH = "C:/Program Files (x86)/MySQL/MySQL Server 5.0/bin/" #question = "Enter the path of the MySQL server if it is not the following name (" + MYSQL_EXE_PATH + "): " #res = input(question) #if res!="": #MYSQL_EXE_PATH = res #VENV_PIP = VENV + VENV_BIN+'pip' ##VENV_BIN = VENV + VENV_BIN+'python' # GLOBAL_PYTHON = 'python3' GLOBAL_PYTHON = os.path.split(sys.executable)[-1] print(Colors.LOG_BLUE + "Python executable is " + GLOBAL_PYTHON + Colors.END) ##if platform.dist()[0] == "centos": print("centos platform") """ ************************************************************************************************* ******************** click package install (if not yet done) and import it ********************** ************************************************************************************************* """ # First thing first, install the click package !!! #import fire try: import click # https://click.palletsprojects.com except: #pip = "pip" if platform.system() == "Windows" else "pip3" # TODO: "python -m pip" au lieu de "pip" pip = "pip" if IS_WINDOWS else "pip3" process = subprocess.Popen(pip + " install --upgrade click", shell=True) process.wait() if process.returncode == 0: print("click package installation successfull") # self.addExecuted(self.current_command, command) import click else: print("click package installation failed") # self.addError(self.current_command, command) """ ************************************************************************** ******************** GENERAL AND UTIL FUNCTIONS ************************** ************************************************************************** """ def _in_dir(dirname: str = ""): return os.path.basename(os.getcwd()) == dirname def _in_abs_dir(dirname: str = ""): return os.getcwd() == dirname def die(msg: str = ""): print() print("...ERROR...") print(msg) print() exit(1) #TODO: implement is_async def execProcess(command, from_venv=False, is_async=False): from_venv_str = " from venv ("+VENV_PYTHON+")" if from_venv else "" #printFullTerm(Colors.BLUE, "Executing command" + " [" + command + "]" + from_venv_str) printd("Executing command" + " [" + command + "]" + from_venv_str) if from_venv: command = VENV_PYTHON+' ' + command process = subprocess.Popen(command, shell=True) if is_async: #current_processes.append(process) return process # Not is_async => Wait for end of execution process.wait() if process.returncode == 0: printFullTerm(Colors.GREEN, "Process executed successfully") # self.addExecuted(self.current_command, command) else: printFullTerm(Colors.WARNING, "Process execution failed") # self.addError(self.current_command, command) #return process.returncode return True if process.returncode==0 else False def execProcessFromVenv(command:str, is_async=False): #return execProcess(command, from_venv=True, is_async) return execProcess(command, True, is_async) #TODO: fusionner dans execProcess avec param is_async def execProcessFromVenvAsync(command:str): return execProcessFromVenv(command, True) """ args = command.split() printFullTerm( Colors.BLUE, "Executing command from venv [" + str(" ".join(args[1:])) + "]" ) p = subprocess.Popen(args) subproc.append((p, " ".join(args[1:]))) printFullTerm(Colors.GREEN, "Process launched successfully") # self.addExecuted(self.current_command, str(' '.join(args[1:]))) # p.wait() return p """ def printColor(color: Colors, message, file=sys.stdout, eol=os.linesep, forced=False): #system = platform.system() """ if (self.disp == False and forced == False): return 0 """ #if system == "Windows": if IS_WINDOWS: print(message, file=file, end=eol) else: print(color + message + Colors.ENDC, file=file, end=eol) return 0 def printFullTerm(color: Colors, string: str): #system = platform.system() columns = 100 row = 1000 disp = True value = int(columns / 2 - len(string) / 2) printColor(color, "-" * value, eol="") printColor(color, string, eol="") value += len(string) printColor(color, "-" * (columns - value)) return 0 """ ******************************************************************************** ******************** CLI COMMANDS DEFINITION (click format) ******************** ******************************************************************************** """ ''' _global_test_options = [ click.option('--test', '-t', is_flag=True, help="don't do it for real, just show what it would do"), click.option('--verbose', '-v', 'verbosity', flag_value=2, default=1, help='Verbose output'), click.option('--quiet', '-q', 'verbosity', flag_value=0, help='Minimal output'), #click.option('--fail-fast', '--failfast', '-f', 'fail_fast', is_flag=True, default=False, help='Stop on failure'), ] def global_test_options(func): for option in reversed(_global_test_options): func = option(func) return func ''' def printd(*args, **kwargs): if os.environ.get('PYROS_DEBUG', '0')=='1': print(*args, **kwargs) GLOBAL_OPTIONS = {} def verbose_mode(): return GLOBAL_OPTIONS["verbose"] def test_mode(): return GLOBAL_OPTIONS["test"] def sim_mode(): return GLOBAL_OPTIONS["sim"] def debug_mode(): return GLOBAL_OPTIONS["debug"] @click.group() @click.option('--debug', '-d', is_flag=True, help='DEBUG mode') @click.option('--sim', '-s', is_flag=True, help="only for an AgentDevice, asking it to start its simulator and work with it (instead of real device)") @click.option('--test', '-t', is_flag=True, help="don't do it for real, just show what it would do") @click.option('--verbose', '-v', is_flag=True, help='Verbose output') #@click.option('--verbose', '-v', 'verbosity', flag_value=2, default=1, help='Verbose output'), #@click.option('--quiet', '-q', 'verbosity', flag_value=0, help='Minimal output'), #@click.option('--fail-fast', '--failfast', '-f', 'fail_fast', is_flag=True, default=False, help='Stop on failure'), def pyros_launcher(debug, sim, test, verbose): #pass os.environ['PYROS_DEBUG'] = '1' if debug else '0' if debug: click.echo('DEBUG mode') if sim: click.echo('Starting and connecting to simulator instead of real device') if test: click.echo('Test mode') if verbose: click.echo('Verbose mode') GLOBAL_OPTIONS["debug"] = debug GLOBAL_OPTIONS["sim"] = sim GLOBAL_OPTIONS["test"] = test GLOBAL_OPTIONS["verbose"] = verbose @pyros_launcher.command(help="Run a pyros shell (django included)") #@global_test_options def shell(): print() print("Launching a pyros shell") print() print("NB1: If you want to play with an agent, type:") print(" >>> from agent.AgentA import AgentA") print(" >>> agent=AgentA('agent_toto')") print(" >>> agent") print(" >>> agent.run(2) (=> will run 2 iterations)") print(" >>> cmd = agent.send_command('AgentB','eval 2+2')") print(" >>> cmd") print(" >>> cmd.get_updated_result()") print(" >>> ...") print(" - See documentation, section 'Play with a pyros agent' inside the chapter 'Running pyros' for more details") print() print("NB2: If you want to play with the pyros objects, type:") print(" >>> from common.models import *") print(" - (This will import all the pyros objects)") print(" - Then, you can create any pyros object just by typing its name.") print(" - For example, to create an AgentSurvey object, type:") print(" >>> agent_survey = AgentSurvey()") print(" >>> agent_survey") print(" >>> ...") print(" - See documentation, section 'Play with the pyros objects' inside the chapter 'Running pyros' for more details") print() print("Type 'exit()' to quit") print() change_dir(PYROS_DJANGO_BASE_DIR) # execProcess("python install.py install") if not test_mode(): execProcessFromVenv("manage.py shell") # Go back to the initial dir #os.chdir("../") change_dir("PREVIOUS") return True @pyros_launcher.command(help="Run a database (Mysql) shell") def dbshell(): print() print("Launching a database (mysql) shell") print("From this shell, you can type 'show tables;' to see all the pyros tables") print("Then for example, type 'select * from config;' to see the content of the 'config' table") print("Type 'exit' to quit") print() # execProcess("python install.py install") if not test_mode(): execProcessFromVenv(PYROS_DJANGO_BASE_DIR+"/manage.py dbshell") # Go back to the initial dir return True @pyros_launcher.command(help="Install the pyros software") @click.option('--packages_only', '-p', is_flag=True, help='install only the python packages (no database installation)') @click.option('--database_only', '-d', is_flag=True, help='install only the pyros database (no python packages installation)') #@global_test_options def install(packages_only, database_only): install_or_update(UPDATE=False, packages_only=packages_only, database_only=database_only) def install_or_update(UPDATE:bool=False, packages_only:bool=False, database_only:bool=False): ACTION = "UPDATING" if UPDATE else "INSTALLING" if not packages_only and not database_only: packages_only = database_only = True if UPDATE: print("Running UPDATE command") else: print("Running INSTALL command") print("- packages_only:", packages_only) print("- database_only:", database_only) #if test_mode(): print("in test mode") # self.execProcess("python3 install/install.py install") # if (os.path.basename(os.getcwd()) != "private"): num = 0 # 1) Update source code (git pull) if UPDATE: num+=1 printFullTerm(Colors.BLUE, f"{num}) UPDATING SOURCE CODE: Running git pull") _gitpull() or die() # 2) Update python packages (pip upgrade AND pip install requirements) if packages_only: num+=1 printFullTerm(Colors.BLUE, f"{num}) {ACTION} PYTHON PACKAGES") # (UPDATE) Re-install VENV if disappeared install_venv(EVEN_IF_ALREADY_EXISTS = not UPDATE) install_packages() # 3) Update PlantUML diagrams num+=1 printFullTerm(Colors.BLUE, f"{num}) UPDATING UML DIAGRAMS") _update_plantuml_diags() or die() print(os.getcwd()) # 4) Install/Update database structure (make migrations + migrate) if database_only: num+=1 printFullTerm(Colors.BLUE, f"{num}) {ACTION} DATABASE") if UPDATE: _updatedb() or die() else: install_database(VENV) print("You can connect to PyROS as 'pyros' user with the password 'DjangoPyros' ") return True @pyros_launcher.command(help="Run some tests") def test(): print("Running tests") #start_dir = os.getcwd() apps = ['common', 'scheduler', 'routine_manager', 'user_manager', 'alert_manager.tests.TestStrategyChange'] for app in apps: _loaddata() or die() change_dir(PYROS_DJANGO_BASE_DIR) # Delete test_pyros database after tests #execProcessFromVenv('manage.py test ' + app) or die() # KEEP test_pyros database after tests execProcessFromVenv('manage.py test --keep ' + app) or die() change_dir("PREVIOUS") # execProcess("python install.py install") return True @pyros_launcher.command(help="Run ALL tests") def testall(): change_dir(PYROS_DJANGO_BASE_DIR) # Delete test_pyros database after tests # execProcessFromVenvAsync("manage.py test") # KEEP test_pyros database after tests execProcessFromVenvAsync("manage.py test --keep") change_dir("PREVIOUS") return True @pyros_launcher.command(help="Update (only if necessary) the python packages AND the source code AND the DB structure") def update(): install_or_update(UPDATE=True) ''' print("Running update command") # 1) Update source code (git pull) printFullTerm(Colors.BLUE, "1) UPDATING SOURCE CODE: Running git pull") _gitpull() or die() # Re-install VENV if disappeared install_venv(False) # 2) Update python packages (pip upgrade AND pip install requirements) printFullTerm(Colors.BLUE, "2) UPDATING PYTHON PACKAGES") ##_update_python_packages_from_requirements() or die() install_packages() print(os.getcwd()) # 3) Update PlantUML diagrams printFullTerm(Colors.BLUE, "3) UPDATING PLANTUML DIAGRAMS") _update_plantuml_diags() or die() print(os.getcwd()) # 4) Update database structure (make migrations + migrate) printFullTerm(Colors.BLUE, "4) UPDATING DATABASE") _updatedb() or die() return True ''' def _gitpull(): print("-- running git pull") GIT = "git.exe" if IS_WINDOWS else "git" if not test_mode(): return execProcess(f"{GIT} pull") return True #@pyros_launcher.command(help="Update the pyros database") def _updatedb(): print("-- update db (make migrations + migrate)") if not test_mode() : _makemigrations() or die() _migrate() or die() return True @pyros_launcher.command(help="Update the pyros database and fill it with initial fixture data") def initdb(): if not test_mode(): #updatedb() _makemigrations() _migrate() _loaddata() return True @pyros_launcher.command(help="Launch an agent") #@global_test_options @click.argument('agent') @click.option('--configfile', '-c', help='the configuration file to be used') #@click.option('--format', '-f', type=click.Choice(['html', 'xml', 'text']), default='html', show_default=True) #@click.option('--port', default=8000) #def start(agent:str, configfile:str, test, verbosity): def start(agent:str, configfile:str): printd("Running start command") if configfile: printd("With config file", configfile) else: configfile = '' #if test_mode(): print("in test mode") #if verbose_mode(): print("in verbose mode") # Check each agent in the given list agents = agent.split(",") if "," in agent else [agent] for a in agents: if not _check_agent(a): return #print("Agents are:", agents) ''' # Check if multiple agents: agents = None if "," in agent: agents = agent.split(",") for a in agents: if not _check_agent(a): return print("Agents are:", agents) # 1 agent only else: agents = [agent] if not _check_agent(agent): return ''' # Start Agents (processes) current_processes = [] for agent_name,agent_folder in AGENTS.items(): #if agent in ("all", agent_name) : if agent=="all" or agent_name in agents: # Default case, launch agentX #if agent_name == "agentX": # execProcessFromVenvAsync(VENV_PYTHON + " manage.py runserver") printd(VENV_PYTHON) printd("Launching agent", agent_name, "...") #if not test_mode(): execProcess(VENV_PYTHON + " manage.py runserver") #if not test_mode(): execProcessFromVenv("start_agent_" + agent_name + ".py " + configfile) current_dir = os.getcwd() # OLD format agents: majordome, monitoring, alert... cmd = "start_agent.py " + agent_name + " " + configfile # Agent "webserver" if agent_name == "webserver": cmd = "manage.py runserver" if(WITH_DOCKER): # If we're running pyros within docker, we need to specify a specific adress in order to access the website on our host machine cmd="manage.py runserver 0.0.0.0:8000" os.chdir(PYROS_DJANGO_BASE_DIR) #if not test_mode(): execProcessFromVenv("start_agent.py " + agent_name + " " + configfile) # Agent "agentM", "agentA", "agentB", "agentX", ... elif agent_name.startswith("agent"): # Run agent without actual commands sent to devices (FOR_REAL=False) ##agentX.run(FOR_REAL=True) if agent_name == "agentM": os.chdir(PYROS_DJANGO_BASE_DIR+"/monitoring/") else: os.chdir(PYROS_DJANGO_BASE_DIR+"/agent/") #cmd = "-m AgentX" #cmd = f" Agent{agent_name[5:]}.py {configfile}" cmd = f"Agent{agent_name[5:]}.py" if debug_mode(): cmd += " -d" if sim_mode(): cmd += " -s" if test_mode(): cmd += " -t" if verbose_mode(): cmd += " -v" if configfile: cmd += " {configfile}" #if not test_mode(): current_processes.append( [execProcessFromVenvAsync(cmd), agent_name, -1] ) # Append this process ( [process id, agent_name, result=failure] ) # ("result" will be updated at the end of execution) current_processes.append( [execProcessFromVenvAsync(cmd), agent_name, -1] ) # self.change_dir("..") os.chdir(current_dir) # Go back to root folder (/) # self.change_dir('..') #os.chdir("..") # Wait for end of each process execution #for (p,agent) in current_processes: for process in current_processes: p,agent,_ = process printd(f"************ Waiting for end of execution of agent {agent} ************") p.wait() process[2] = p.returncode print(f"************ END of execution of agent {agent} ************") if p.returncode == 0: printFullTerm(Colors.GREEN, f"Process {agent} executed successfully") # self.addExecuted(self.current_command, command) else: printFullTerm(Colors.WARNING, f"Process {agent} execution failed") # self.addError(self.current_command, command) print() print() print("Synthesis of the results:") for process in current_processes: p,agent,returncode = process if returncode == 0: printFullTerm(Colors.GREEN, f"Process {agent} executed successfully") # self.addExecuted(self.current_command, command) else: printFullTerm(Colors.WARNING, f"Process {agent} execution failed") # self.addError(self.current_command, command) #print("************ end of START() ************") # Only according to the last process status: #return True if p.returncode==0 else False return True if p.returncode==0 else False # TODO: implémenter le STOP !!! @pyros_launcher.command(help="Kill an agent") @click.argument('agent') def stop(agent): print("Running stop command") if not _check_agent(agent): return """ ******************************************************************************** ******************** PRIVATE FUNCTIONS DEFINITION ****************************** ******************************************************************************** """ def notused_update_python_packages_from_requirements(): # 1) Upgrade pip (if new version available) res = execProcessFromVenv("-m pip install --upgrade pip") if not res: return False # 2) Install only "not yet installed" python packages res = execProcessFromVenv("-m pip install -r install/"+REQUIREMENTS) return res def _update_plantuml_diags(): for dirpath, dirnames, files in os.walk(PYROS_DJANGO_BASE_DIR): if os.path.basename(dirpath) == "doc": diagrams = glob.glob(dirpath+os.sep+"*.pu") # For each diagram source file (.pu), generate the diagram PNG file (only if CHANGED) for diag in diagrams: # if no diag.png or if diag.pu more recent than diag.png => re-generate diag.png diag_png = diag.split('.pu')[0] + '.png' if not os.path.isfile(diag_png) or ( os.path.getmtime(diag) > os.path.getmtime(diag_png) ) : #res = execProcessFromVenv("-m plantuml "+diag) # we need to use our own version of plantuml res = execProcessFromVenv("./install/plantuml.py "+diag) if not res: return False return True def _migrate(): change_dir(PYROS_DJANGO_BASE_DIR) # Migrate only migrations for the app "common" #res = execProcessFromVenv("manage.py migrate common") # Migrate all migrations for ALL apps res = execProcessFromVenv("manage.py migrate") change_dir("PREVIOUS") return res def _makemigrations(): change_dir(PYROS_DJANGO_BASE_DIR) #execProcessFromVenv(self.venv_bin + " manage.py makemigrations") #res = execProcessFromVenv("manage.py makemigrations common") res = execProcessFromVenv("manage.py makemigrations") change_dir("PREVIOUS") return res #TODO: mettre la fixture en date naive (sans time zone) def _loaddata(): change_dir(PYROS_DJANGO_BASE_DIR) #execProcessFromVenv(self.venv_bin + " manage.py loaddata misc" + os.sep + "fixtures" + os.sep + self.INIT_FIXTURE) res = execProcessFromVenv("manage.py loaddata misc" + os.sep + "fixtures" + os.sep + INIT_FIXTURE) change_dir("PREVIOUS") return res def change_dir(path): global _previous_dir if path == "PREVIOUS": path=_previous_dir _previous_dir = os.getcwd() printd("Moving to : " + path) os.chdir(path) printd("Current directory : " + str(os.getcwd())) def _check_agent(agent): # Check that agent exists if agent != "all" and agent not in AGENTS.keys() : # Check if multiple agents: print("This agent does not exist") print("Here is the allowed list of agents:") print("- all => will launch ALL agents") for agent_name in AGENTS.keys(): print('-',agent_name) return False return True """ INSTALLATION FUNCTIONS (./pyros.py install) """ def replacePatternInFile(pattern, replace, file_path): try: with fileinput.FileInput(file_path, inplace=True, backup='.bak') as file: for line in file: print(line.replace(pattern, replace), end='') except: sys.stderr.write(Colors.ERROR + "ERROR !: replacement in file failed !" + Colors.END + "\r\n") return 1 return 0 def notused_install_dependency_ubuntu(command, mode): ''' Install dependency then check the return code ''' old = command if (mode == 'i'): command = 'apt-get install ' + command elif (mode == 'u'): command = 'apt-get update' elif (mode == 'a'): command = 'add-apt-repository ' + command process = subprocess.Popen(command, shell=True) process.wait() if process.returncode != 0: sys.stderr.write(Colors.ERROR + "ERROR !: installation of " + old + " failed !" + Colors.END + "\r\n") def notused_install_required_ubuntu(): install_dependency_ubuntu("update", 'u') install_dependency_ubuntu("python-lxml", 'i') install_dependency_ubuntu("libxml2-dev", 'i') install_dependency_ubuntu("libxslt-dev", 'i') install_dependency_ubuntu("zlib1g-dev", 'i') install_dependency_ubuntu("update", 'u') install_dependency_ubuntu("rabbitmq-server", 'i') #install_dependency_ubuntu("libmysqlclient-dev", 'i') def notused_install_dependency_centos(command, mode): old = command if (mode == 'i'): command = 'yum -y install ' + command elif (mode == 'u'): command = 'yum update ' + command process = subprocess.Popen(command, shell=True) process.wait() if process.returncode != 0: sys.stderr.write(Colors.ERROR + "ERROR !: installation of " + old + " failed !" + Colors.END + "\r\n") def notused_install_required_centos(): install_dependency_centos("yum", 'u') install_dependency_centos("kernel", 'u') install_dependency_centos("", 'u') install_dependency_centos("libxml2", 'i') install_dependency_centos("libxslt libxslt-2", 'i') install_dependency_centos("libxslt-devel libxml2-devel", 'i') install_dependency_centos("rabbitmq-server", 'i') install_dependency_centos("mariadb-server", 'i') install_dependency_centos("mariadb", 'i') install_dependency_centos("mariadb-devel", 'i') process = subprocess.Popen("systemctl start mariadb.service", shell=True) process.wait() if process.returncode != 0: sys.stderr.write(Colors.ERROR + "ERROR !" + Colors.END + "\r\n") process = subprocess.Popen("systemctl enable mariadb.service", shell=True) process.wait() if process.returncode != 0: sys.stderr.write(Colors.ERROR + "ERROR !" + Colors.END + "\r\n") process = subprocess.Popen("mysql_secure_installation", shell=True) process.wait() if process.returncode != 0: sys.stderr.write(Colors.ERROR + "ERROR !" + Colors.END + "\r\n") def notused_install_required(): # Checking if user is sudo then install the needed dependencies # Find the linux distribution and call the related function distribution = platform.dist() if not 'SUDO_UID' in os.environ.keys(): sys.stderr.write("Super user rights are needed to install prerequisites\r\n") exit(1) if distribution[0] == "Ubuntu" or distribution[0] == "Debian": install_required_ubuntu() elif distribution[0] == "centos": install_required_centos() else: print("Requirements are made for Ubuntu, Debian and CentOS only") exit(1) def venv_pip_install(package_name:str, options:str=''): os.system(VENV_PIP + ' install ' + options + ' ' + package_name) def install_venv(EVEN_IF_ALREADY_EXISTS:bool=False): ''' Install VENV if does not exist OR Re-install it if EVEN_IF_ALREADY_EXISTS is true (return true if no issue) ''' # -------------------------------------------- # --- Be aware not to create virtual environment in case of user root # -------------------------------------------- if 'SUDO_UID' in os.environ.keys(): answer = input( "You are about to install your virtualenv only for root, this is discouraged, are you sure ? (Y/N) If you are not sure, relaunch the script without super user privileges\n") while (answer != 'Y' and answer != 'y' and answer != 'n' and answer != 'N'): answer = input( "You are about to install your virtualenv only for root, this is discouraged, are you sure ? (Y/N) \n") if (answer not in ['y', 'Y']): exit(1) # -------------------------------------------- # --- (If not exists) Create the (private) venv ROOT directory (where the virtual environment dir will be created) # -------------------------------------------- if (os.path.basename(os.getcwd()) != VENV_ROOT): ##if not(os.path.isdir("../venv")): if not(os.path.isdir(VENV_ROOT)): print(Colors.LOG_BLUE + f"-----------------------------Creating 'venv' root directory ({VENV_ROOT}/)-----------------------------" + Colors.END) #os.mkdir("../venv") os.mkdir(VENV_ROOT) os.chdir(VENV_ROOT) # -------------------------------------------- # --- (if EVEN_IF_ALREADY_EXISTS) Delete venv dir if already exist # -------------------------------------------- #print(Colors.LOG_BLUE + "-----------------------------cd venv-----------------------------" + Colors.END) if EVEN_IF_ALREADY_EXISTS: while True: try: if os.path.isdir(VENV): print(Colors.LOG_BLUE + f"-----------------------------Deleting existing VENV dir ({VENV}/)-----------------------------" + Colors.END) shutil.rmtree(VENV) break # Exception on Windows WinError 145 : Cannot remove folder because files in folder not yet removed... except Exception as e: #print(e) continue # -------------------------------------------- # --- (if not exists) Reinstall the virtual environment (from ../venv/) # -------------------------------------------- if not os.path.isdir(VENV): print(Colors.LOG_BLUE + f"-----------------------------Creating venv dir ({VENV}/)-----------------------------"+END_OF_LINE + Colors.END) os.system(GLOBAL_PYTHON+" -m venv " + VENV) # Come back to project root os.chdir('..') return True def install_packages(): os.chdir(VENV_ROOT) print(Colors.LOG_BLUE + "-----------------------------Upgrade pip, wheel, and setuptools" + "-----------------------------"+END_OF_LINE + Colors.END) # Upgrade pip os.system(VENV_PYTHON + ' -m pip install --upgrade pip') ''' if (platform.system() == "Windows"): os.system(venv + '\Scripts\python -m pip install --upgrade pip') else: # Linux os.system(venv + '/bin/python -m pip install --upgrade pip') ''' # Pip upgrade wheel and setuptools venv_pip_install('wheel', '--upgrade') #os.system(VENV_PIP+' install --upgrade wheel') venv_pip_install('setuptools', '--upgrade') #os.system(VENV_PIP+' install --upgrade setuptools') # Pip install required packages from REQUIREMENTS file print() print(Colors.LOG_BLUE + "-----------------------------Installing python packages via pip-----------------------------" + Colors.END) venv_pip_install('../install/'+REQUIREMENTS, '-r') #os.system(VENV_PIP+' install -r ../install' + os.sep + REQUIREMENTS) #print(Colors.LOG_BLUE + "-----------------------------cd ../install-----------------------------" + Colors.END) if IS_WINDOWS: os.chdir("../install") ## moving voeventparse in site-packages directory try: site_packages = "..\\venv\\"+VENV+"\\Lib\\site-packages\\" if ( not os.path.isdir(site_packages + "voevent_parse-0.9.5.dist-info") and not os.path.isdir(site_packages + "voeventparse") ): print(Colors.LOG_BLUE + "\r\n\r\n-----------------------------Copying the voevent library in Lib/site-packages-----------------------------" + Colors.END) cmdline = "xcopy /i /y windows\\voeventparse " + site_packages + "voeventparse" process = subprocess.Popen(cmdline) process.wait() if (process.returncode != 0): raise Exception process = subprocess.Popen("xcopy /i /y windows\\voevent_parse-0.9.5.dist-info " + site_packages + "voevent_parse-0.9.5.dist-info") process.wait() if (process.returncode != 0): raise Exception print(Colors.LOG_BLUE + "\r\n-----------------------------library successfully copied-----------------------------" + Colors.END) except Exception as e: print(Colors.ERROR + "ERROR while Copying the voevent library in Lib/site-packages" + Colors.END) ; #, file=stderr) return False # Go back to project root dir os.chdir('..') #return 0 return True def install_database(venv): print(Colors.LOG_BLUE + END_OF_LINE+"-----------------------------Launching mysql to create database and create and grant user pyros-----------------------------" + Colors.END) # -------------------------------------------- # --- Determine the MySQL version # -------------------------------------------- output = subprocess.check_output("mysql --version", shell=True) # output is something like: "mysql Ver 15.1 Distrib 10.0.20-MariaDB, for Linux (x86_64) using EditLine wrapper" tmp = (str(output).split()[4]).split('.') sql_version = float(tmp[0]+'.'+tmp[1]) print(Colors.LOG_BLUE + "MySQL version is " + str(sql_version) + Colors.END) # -------------------------------------------- # --- Prepare the SQL query to create and initialize the pyros database if needed # -------------------------------------------- ''' (OLD) sql_query = \ f"CREATE DATABASE IF NOT EXISTS {SQL_DATABASE}; " +\ f"CREATE DATABASE IF NOT EXISTS {SQL_DATABASE_SIMU}; " if sql_version < 5.5: #sql_query = "drop database "+SQL_DATABASE+" ; CREATE DATABASE "+SQL_DATABASE+"; drop database "+SQL_DATABASE_SIMU+" ; CREATE DATABASE "+SQL_DATABASE_SIMU+"; CREATE USER "+SQL_USER+" ; GRANT USAGE ON *.* TO '"+SQL_USER+"'@'localhost' IDENTIFIED BY '"+SQL_PSWD+"' WITH GRANT OPTION; DROP USER '"+SQL_USER+"'@'localhost'; GRANT ALL ON "+SQL_DATABASE+".* TO '"+SQL_USER+"'@'localhost' IDENTIFIED BY '"+SQL_PSWD+"'; GRANT ALL PRIVILEGES ON "+SQL_DATABASE+".* TO '"+SQL_USER+"'@'localhost' IDENTIFIED BY '"+SQL_PSWD+"' WITH GRANT OPTION; GRANT ALL PRIVILEGES ON "+SQL_DATABASE_SIMU+".* TO "+SQL_USER+"@localhost IDENTIFIED BY '"+SQL_PSWD+"' WITH GRANT OPTION;" sql_query = \ f"DROP DATABASE {SQL_DATABASE}; CREATE DATABASE {SQL_DATABASE}; " +\ f"DROP DATABASE {SQL_DATABASE_SIMU}; CREATE DATABASE {SQL_DATABASE_SIMU}; " sql_query += "CREATE USER "+SQL_USER+" ; GRANT USAGE ON *.* TO '"+SQL_USER+"'@'localhost' IDENTIFIED BY '"+SQL_PSWD+"' WITH GRANT OPTION; " sql_query += "DROP USER '"+SQL_USER+"'@'localhost'; " sql_query += "GRANT ALL ON "+SQL_DATABASE+".* TO '"+SQL_USER+"'@'localhost' IDENTIFIED BY '"+SQL_PSWD+"'; " sql_query += "GRANT ALL PRIVILEGES ON "+SQL_DATABASE+".* TO '"+SQL_USER+"'@'localhost' IDENTIFIED BY '"+SQL_PSWD+"' WITH GRANT OPTION; " sql_query += "GRANT ALL PRIVILEGES ON "+SQL_DATABASE_SIMU+".* TO "+SQL_USER+"@localhost IDENTIFIED BY '"+SQL_PSWD+"' WITH GRANT OPTION; " ''' # 1) Create databases pyros and pyros_test (but not test_pyros because will be automatically created by django) # TODO: Pour mysql < 5.5, comment éviter un "drop database" inutile (si la BD n'existe pas encore) qui va provoquer un plantage mysql ? IF_EXISTS = '' if sql_version < 5.5 else 'IF EXISTS' sql_query = \ f"DROP DATABASE {IF_EXISTS} {SQL_DATABASE}; CREATE DATABASE {SQL_DATABASE}; " +\ f"DROP DATABASE {IF_EXISTS} {SQL_DATABASE_SIMU}; CREATE DATABASE {SQL_DATABASE_SIMU}; " # 2) Create user pyros and give it all rights on 3 databases # Ne marche pas si l'utilisateur existe déjà => erreur #"CREATE USER "+SQL_USER+"; " +\ # Donc, il faut ruser # Mysql >= 5.7 only: #sql_query_elem = "CREATE USER IF NOT EXISTS "+SQL_USER+"; " # Si user n'existe pas => est créé ; Si user existe => pas d'erreur ; DONC ok dans les 2 cas #sql_query_elem = "GRANT ALL ON "+SQL_DATABASE+".* TO '"+SQL_USER+"'@'localhost' IDENTIFIED BY '"+SQL_PSWD+"'; " if WITH_DOCKER: # we will use this later (at the end of development) #host = os.environ["IP_PYROS_USER"] host = "%" else: host = "localhost" sql_query += f"GRANT ALL ON {SQL_DATABASE}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; " sql_query += f"GRANT ALL ON {SQL_DATABASE_SIMU}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; " # This database does not yet exists and will be automatically created by Django, but we already give access rights to it for pyros user sql_query += f"GRANT ALL ON {SQL_DATABASE_TEST}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; " #"GRANT USAGE ON *.* TO '"+SQL_USER+"'@localhost ; " ''' # (EP) AVANT, y avait tout ça..., vraiment utile ? "GRANT USAGE ON *.* TO '"+SQL_USER+"'; " +\ "DROP USER '"+SQL_USER+"'; " +\ "GRANT ALL ON "+SQL_DATABASE+".* TO '"+SQL_USER+"'@'localhost' IDENTIFIED BY '"+SQL_PSWD+"'; " +\ "GRANT ALL ON "+SQL_DATABASE_SIMU+".* TO '"+SQL_USER+"'@'localhost'; " +\ "GRANT ALL PRIVILEGES ON "+SQL_DATABASE_SIMU+".* TO '"+SQL_USER+"'@'localhost'; " +\ "GRANT ALL ON "+SQL_DATABASE_SIMU+".* TO '"+SQL_USER+"'@'localhost' IDENTIFIED BY '"+SQL_PSWD+"' ;" ''' # NEWER MYSQL: # OLDER MYSQL: Try this instead for OLDER mysql (works on CentOS 6.4 and Centos 7.5 with mysql 5.5): #req = "drop database pyros; CREATE DATABASE pyros; drop database pyros_test ; CREATE DATABASE pyros_test; drop user 'pyros'@'localhost' ; CREATE USER pyros; GRANT USAGE ON *.* TO 'pyros'; DROP USER 'pyros'; GRANT ALL ON pyros.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros'; GRANT ALL ON test_pyros.* TO 'pyros'@'localhost'; GRANT ALL PRIVILEGES ON test_pyros_test.* TO 'pyros'@'localhost'; GRANT ALL ON pyros_test.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros'" #req = "drop database pyros ; CREATE DATABASE pyros; drop database pyros_test ; CREATE DATABASE pyros_test; DROP USER 'pyros'@'localhost' ; GRANT USAGE ON *.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros' WITH GRANT OPTION; DROP USER 'pyros'@'localhost'; GRANT ALL ON pyros.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros'; GRANT ALL PRIVILEGES ON pyros.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros' WITH GRANT OPTION; GRANT ALL PRIVILEGES ON pyros_test.* TO pyros@localhost IDENTIFIED BY 'DjangoPyros' WITH GRANT OPTION;" # (EP) ok for CENTOS 7 I suppose (but not for CentOS 6): #req_centos = "CREATE DATABASE IF NOT EXISTS pyros; CREATE DATABASE IF NOT EXISTS pyros_test; GRANT USAGE ON *.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros' WITH GRANT OPTION; DROP USER 'pyros'@'localhost'; GRANT ALL ON pyros.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros'; GRANT ALL PRIVILEGES ON pyros.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros' WITH GRANT OPTION; GRANT ALL PRIVILEGES ON pyros_test.* TO pyros@localhost IDENTIFIED BY 'DjangoPyros' WITH GRANT OPTION;" # --- Prepare the SQL query to create and initialize the pyros database if needed #if platform.dist()[0] == "centos": # req = sql_query #mysql_call_root = '"' + MYSQL_EXE_PATH + 'mysql" -u root -p' # if the mysql_root_password isn't defined, it will ask to enter the password try: # a root password should be defined as an environment variable mysql_root_password = os.environ['MYSQL_ROOT_PASSWORD'] except: # need a space as string so the command ask to the user to write his password. mysql_root_password = " " if WITH_DOCKER: # host is the name of the service of database in docker-compose (i.e. host=db) mysql_host = "db" else: mysql_host = "localhost" # when using -p option, there is no space between the option and the password value mysql_call_root = f'mysql -h {mysql_host} -u root -p{mysql_root_password}' mysql_call_pyros = "\"" + MYSQL_EXE_PATH+ "mysql\" -u "+SQL_USER+" -p" # -------------------------------------------- # --- Creating database and creating and granting user pyros # -------------------------------------------- user_ros_is_created = True if sql_version<5.5: print(Colors.LOG_BLUE +"------------------ Check if the user pyros exists in MYSQL (type the pyros password) -----------------------------" + Colors.END) # --- We are testing if user pyros already exists in the database process = subprocess.Popen("echo quit |" + mysql_call_pyros, shell=True) process.wait() if (process.returncode == 0): user_ros_is_created = False if user_ros_is_created: # --- The user pyros must be created in the database print(Colors.LOG_BLUE +"-----------------------------Please enter your MYSQL root password-----------------------------" + Colors.END) #process = subprocess.Popen("echo \"" + sql_query + "\" |"+ mysql_call_root, shell=True) sql_cmd = 'echo "' + sql_query + '" | '+ mysql_call_root print("Executing sql cmd: ", sql_cmd) process = subprocess.Popen(sql_cmd, shell=True) process.wait() if (process.returncode != 0): sys.stderr.write(Colors.ERROR + "ERROR !: db configuration failed !" + Colors.END + "\r\n") return -1 print(Colors.LOG_BLUE + END_OF_LINE+"-----------------------------Database created and user pyros successfully created and granted-----------------------------" + Colors.END) # -------------------------------------------- # --- Replacing pattern in settings.py to use mysql # -------------------------------------------- print(Colors.LOG_BLUE + "-----------------------------setting MYSQL = True in settings-----------------------------" + Colors.END) replacePatternInFile("MYSQL = False", "MYSQL = True", os.path.normpath("src/core/pyros_django/pyros/settings.py")) #print(Colors.LOG_BLUE + "\r\n-----------------------------cd ..-----------------------------" + Colors.END) # 2021 : no need to change folder in order to init the database (maybe the next line was forget to be commented because the previous line is commented) #os.chdir("..") # -------------------------------------------- # --- Executing migrations # -------------------------------------------- print(Colors.LOG_BLUE + "\r\n\r\n-----------------------------Migrate : executing pyros.py init_database-----------------------------" + Colors.END) #TODO: from venv !!! try: #os.system(GLOBAL_PYTHON+" pyros.py init_database") os.system(GLOBAL_PYTHON+" pyros.py initdb") ''' process = subprocess.Popen(GLOBAL_PYTHON + " pyros.py init_database" , shell=True) process.wait() ''' except Exception as e: print("Exception ", e) print(Colors.ERROR + "Error while initializing database :" + Colors.END) return -1 print(Colors.LOG_BLUE + "\r\n\r\n-----------------------------Install successfull !-----------------------------" + Colors.END) return 0 def notused_help(): print( #"Welcome in the installation script of the pyros venv.\t\nPlease launch it from the install directory of pyros.\n\tIf you're on Ubuntu Debian or CentOS:\n\tlaunch it with sudo and <--prerequisites> or <-p> to install the prerequisites.\n\t-->sudo ./test_install.py -p\n\n\tFor the python packages launch it from the install directory of pyros without sudo and without parameter\n\t-->./test_install.py") "Welcome to the installation script of the pyros software.\t\n" + \ "Usage:\n" + \ " [python] ./pyros.py install [-p] [-d]" ) """ ******************************************************************************** ********************************* main() FUNCTION ****************************** ******************************************************************************** """ def main(): pyros_launcher() # AVIRER #@click.command() #@click.argument('start') def oldmain(): ''' cli() return ''' #fire.Fire(Commands) #return # if len(sys.argv) == 3 and sys.argv[1].startswith("simulator"): SIMULATOR_CONFIG_FILE = sys.argv[2] # print(sys.argv) """ system = platform.system() columns = 100 row = 1000 disp = True """ # Read command if len(sys.argv) <= 1: die("You must give a command name") command = sys.argv[1] if not command in COMMANDS.keys(): die("This command does not exist") command_all_args = COMMANDS[command] # Read command args if should be command_arg = None if command_all_args: if len(sys.argv) <= 2: die("This command should be given an argument") command_arg = sys.argv[2] if not command_arg in command_all_args: die("This argument does not exist for command " + command) print("Executing command", command) if command_arg: print("with arg", command_arg) # command(command_arg) if command_arg: globals()[command](command_arg) else: globals()[command]() # sys.exit(pyros.exec()) if __name__ == "__main__": main() """ INSTALLATION main function if __name__ == '__main__': if len(sys.argv) > 2: _help() ; sys.exit(1) #print(sys.argv) if len(sys.argv) == 2: INSTALL_VENV = INSTALL_DB = False if sys.argv[1] == '-p': INSTALL_VENV=True if sys.argv[1] == '-d': INSTALL_DB=True ''' # Prerequisistes installation (for CentOS) => not a good idea for now, deactivated (EP) if sys.argv[1] == "--prerequisites" or sys.argv[1] == "-p": install_required() else: _help() ''' if INSTALL_VENV: install_venv(VENV) if INSTALL_DB: install_database(VENV) """