#!/usr/bin/env python3 ''' ******************************************************* ******************** IMPORTS ************************** ******************************************************* ''' ''' *** STANDARD IMPORTS *** ''' # import logging ''' *** PROJECT IMPORTS *** ''' # from src.pyros_logger import * # from src.pyros_logger import log, log_d,log_i,log_w,log_e,log_c # from src import pyros_logger """ ***************************************************************** ******************** GENERAL CONSTANTS ************************** ***************************************************************** """ # DEBUG = False from src.pyros_logger import log import re import time import sys import subprocess import signal import shutil import platform import os import glob import fileinput import argparse UPDATE = False # install dev packages too ? DEV = True PYROS_DJANGO_BASE_DIR = "src/core/pyros_django" if DEV: INIT_FIXTURE = "initial_fixture_dev_TZ.json" else: INIT_FIXTURE = "initial_fixture_TZ.json" _previous_dir = None # List of all agents that pyros can start, with this format : # agent-name : agent-folder-name AGENTS = { "AgentM": "monitoring", "AgentSP": "scientific_programs", "AgentScheduler": "scheduler", "agentImagesProcessor": "observation_manager", "AgentImagesCalibrator": "observation_manager", "agentSST": "agent", "Agent": "agent", "Agent2": "agent", "AgentX": "agent", "AgentA": "agent", "AgentB": "agent", "AgentC": "agent", # "agentDevice" : "AgentDevice", # "agentDeviceTelescopeGemini" : "AgentDeviceTelescopeGemini", "AgentDeviceGemini": "AgentDeviceGemini", "AgentDeviceSBIG": "AgentDeviceSBIG", "AgentTelescopeRequester": "AgentTelescopeRequester", "AgentMultiRequester": "AgentMultiRequester", "webserver": "webserver", "monitoring": "monitoring", "majordome": "majordome", #"scheduler": "scheduler", "alert_manager": "alert_manager", "AgentImagesProcessor_tnc_up1": "../../../privatedev/plugin/agent", "agentBasic": "agent", "AgentTriton": "agent", "AgentMCO":"agent", } # AGENTS = ["agentX", "webserver", "monitoring", "majordome", "scheduler", "alert_manager"] # AGENTS = ["all", "webserver", "monitoring", "majordome", "scheduler", "alert"] # COMMANDS = {"install": [], "start": AGENTS, "stop": AGENTS} REQUIREMENTS = 'requirements.txt' REQUIREMENTS_DEV = 'requirements_dev.txt' b_in_dir = "bin" PYTHON = "python3" # should also be ok from venv: # PYTHON = "python" IS_WINDOWS = platform.system() == "Windows" if IS_WINDOWS: REQUIREMENTS = 'requirements_win.txt' REQUIREMENTS_DEV = 'requirements_win_dev.txt' b_in_dir = "Scripts" PYTHON = "python.exe" PROJECT_ROOT_PATH = os.getcwd() os.environ["PROJECT_ROOT_PATH"] = PROJECT_ROOT_PATH try: # WITH_DOCKER is an environment varialbe from our Docker image WITH_DOCKER = os.environ['WITH_DOCKER'] except KeyError: WITH_DOCKER = False if type(WITH_DOCKER) is str and re.match("^y$|^Y$|^yes$|^Yes$", WITH_DOCKER.rstrip()) != None: WITH_DOCKER = True else: WITH_DOCKER = False my_abs_path = os.path.dirname(os.path.realpath(__file__)) # VENV_ROOT = "private" if WITH_DOCKER: VENV_ROOT = "" VENV = "" VENV_BIN = "" else: VENV_ROOT = "venv" VENV = "venv_py3_pyros" VENV_BIN = ( my_abs_path + os.sep + VENV_ROOT + os.sep + VENV + os.sep + b_in_dir + os.sep ) VENV_PYTHON = VENV_BIN + PYTHON VENV_PIP = VENV_BIN + 'pip' class Colors: HEADER = "\033[95m" BLUE = "\033[94m" GREEN = "\033[92m" WARNING = "\033[93m" FAIL = "\033[91m" ENDC = "\033[0m" BOLD = "\033[1m" UNDERLINE = "\033[4m" ERROR = '\033[91m' END = '\033[0m' LOG_BLUE = '\033[94m' if IS_WINDOWS: ERROR = '' END = '' LOG_BLUE = '' ''' VARIABLES FOR INSTALL ''' # By default, install the virtual environment AND the database INSTALL_VENV = True INSTALL_DB = True SQL_DATABASE = "pyros" # Database automatically created by Django for tests (and automatically deleted after tests) SQL_DATABASE_TEST = "test_pyros" # Specific database that we need(ed?) for some simulations SQL_DATABASE_SIMU = "pyros_test" SQL_USER = "" SQL_PSWD = "" MYSQL_EXE_PATH = "" ENV_PATH = "docker/variables.env" ENV_SAMPLE_PATH = "docker/.env-sample" END_OF_LINE = '\n\n' # VENV_BIN = '/bin/' # -------------------------------------------- # --- Modified values for Windows # -------------------------------------------- if (platform.system() == "Windows"): END_OF_LINE = "\r\n\r\n" # VENV_BIN = '\\Scripts\\' # MYSQL_EXE_PATH = "C:/Program Files (x86)/MySQL/MySQL Server 5.0/bin/" # question = "Enter the path of the MySQL server if it is not the following name (" + MYSQL_EXE_PATH + "): " # res = input(question) # if res!="": # MYSQL_EXE_PATH = res # VENV_PIP = VENV + VENV_BIN+'pip' # VENV_BIN = VENV + VENV_BIN+'python' # GLOBAL_PYTHON = 'python3' GLOBAL_PYTHON = os.path.split(sys.executable)[-1] log.debug(Colors.LOG_BLUE + "Python executable is " + GLOBAL_PYTHON + Colors.END) # if platform.dist()[0] == "centos": print("centos platform") """ ************************************************************************************************* ******************** click package install (if not yet done) and import it ********************** ************************************************************************************************* """ # First thing first, install the click package !!! # import fire try: import click # https://click.palletsprojects.com except: # pip = "pip" if platform.system() == "Windows" else "pip3" # TODO: "python -m pip" au lieu de "pip" pip = "pip" if IS_WINDOWS else "pip3" process = subprocess.Popen(pip + " install --upgrade click", shell=True) process.wait() if process.returncode == 0: log.debug("click package installation successfull") # self.addExecuted(self.current_command, command) import click else: log.error("click package installation failed") # self.addError(self.current_command, command) """ ************************************************************************** ******************** GENERAL AND UTIL FUNCTIONS ************************** ************************************************************************** """ def _in_dir(dirname: str = ""): return os.path.basename(os.getcwd()) == dirname def _in_abs_dir(dirname: str = ""): return os.getcwd() == dirname def die(msg: str = ""): log.error('') log.error("...ERROR...") log.error(msg) log.error('') exit(1) # TODO: implement is_async def execProcess(command, from_venv=False, is_async=False,foreground=False): from_venv_str = " from venv ("+VENV_PYTHON+")" if from_venv else "" # printFullTerm(Colors.BLUE, "Executing command" + " [" + command + "]" + from_venv_str) log.debug( "Executing command" + " [" + command + "]" + from_venv_str ) if from_venv: command = VENV_PYTHON+' ' + command print(f"EXECUTING {command}") if foreground: process = subprocess.Popen(command, shell=True) else: process = subprocess.Popen(command, shell=True,stdout=subprocess.DEVNULL,stderr=subprocess.STDOUT) if is_async: # current_processes.append(process) return process # Not is_async => Wait for end of execution process.wait() if process.returncode == 0: printFullTerm(Colors.GREEN, "Process executed successfully") # self.addExecuted(self.current_command, command) else: printFullTerm(Colors.WARNING, "Process execution failed") # self.addError(self.current_command, command) # return process.returncode return True if process.returncode == 0 else False def execProcessFromVenv(command: str, is_async=False,foreground=True): # return execProcess(command, from_venv=True, is_async) return execProcess(command, True, is_async,foreground) # TODO: fusionner dans execProcess avec param is_async def execProcessFromVenvAsync(command: str,foreground=False): return execProcessFromVenv(command, True,foreground) """ args = command.split() printFullTerm( Colors.BLUE, "Executing command from venv [" + \ str(" ".join(args[1:])) + "]" ) p = subprocess.Popen(args) subproc.append((p, " ".join(args[1:]))) printFullTerm(Colors.GREEN, "Process launched successfully") # self.addExecuted(self.current_command, str(' '.join(args[1:]))) # p.wait() return p """ def printColor(color: Colors, message, file=sys.stdout, eol=os.linesep, forced=False): # system = platform.system() """ if (self.disp == False and forced == False): return 0 """ # if system == "Windows": if IS_WINDOWS: print(message, file=file, end=eol) else: print(color + message + Colors.ENDC, file=file, end=eol) return 0 def printFullTerm(color: Colors, string: str): # system = platform.system() columns = 100 row = 1000 disp = True value = int(columns / 2 - len(string) / 2) printColor(color, "-" * value, eol="") printColor(color, string, eol="") value += len(string) printColor(color, "-" * (columns - value)) return 0 def set_environment_variables_if_not_configured(env_path: str, env_sample_path: str) -> None: """ Set environment variables if they aren't defined in the current environment. Get the variables from .env file if it exists or create this file using a copy of the env_sample Args: env_path (str): path to .env file env_sample_path (str): path to .env-sample file """ """ variables_names = ["MYSQL_ROOT_PASSWORD","MYSQL_ROOT_LOGIN","MYSQL_TCP_PORT", "MYSQL_PYROS_LOGIN","MYSQL_PYROS_PWD","PATH_TO_OBSCONF_FILE"] is_environment_variables_defined = True while(is_environment_variables_defined): for variable in variables_names: if( os.environ.get(variable) is None): is_environment_variables_defined = False break if(not is_environment_variables_defined): print("Some environment variables are not configured...") """ log.debug("Start Setting environment variables") try: with open(env_path, "r") as env_file: # env file is empty if os.stat(env_path).st_size == 0: raise BaseException() log.debug("Reading env file") for line in env_file: if(line.startswith("#") and not line.strip()): continue else: line = line.strip() key, value = line.split("=") key = key.strip() value = value.strip() # setting variables as environment variables if WITH_DOCKER and os.environ.get(key) != value: log.warning( f"WARNING: Environment value for '{key}' needs to be uptaded (To remove this message : restart docker container to update them).'") else: # if not WITH_DOCKER and os.environ.get(key) is None: # log.error(f"ERROR: Error while reading value for '{key}'.Please fix the '{env_file}' file. For this run, PyROS will take the values from '{env_path}'") # raise BaseException() if WITH_DOCKER and os.environ.get(key) is None: log.warning( f"WARNING: Environment value for '{key}' isn't defined within the container (To remove this message : restart docker container to update environment values). PyROS will take the values from '{env_path}'") raise BaseException() os.environ[key] = value except: # print(f".env not found at {env_path} or is empty, creating a file at this path from the .env-sample file stored at {ENV_SAMPLE_PATH}\nvalues from .env-sample will be used as environment variables") with open(env_sample_path, 'r') as env_sample_file: with open(env_path, "w") as env_file: for env_sample_line in env_sample_file: if(env_sample_line.startswith("#") or not env_sample_line.strip()): continue key, value = env_sample_line.split("=") os.environ[key] = value env_file.write(env_sample_line) # PYROS_DJANGO_BASE_DIR isn't an absolute path so we need to add the path to current folder to it os.environ["DJANGO_PATH"] = os.path.join( os.getcwd(), PYROS_DJANGO_BASE_DIR) log.debug("End Setting environment variables") """ else: print("The environment variables are already configured, skipping this step...") """ """ ******************************************************************************** ******************** CLI COMMANDS DEFINITION (click format) ******************** ******************************************************************************** """ ''' _global_test_options = [ click.option('--test', '-t', is_flag=True, help="don't do it for real, just show what it would do"), click.option('--verbose', '-v', 'verbosity', flag_value=2, default=1, help='Verbose output'), click.option('--quiet', '-q', 'verbosity', flag_value=0, help='Minimal output'), # click.option('--fail-fast', '--failfast', '-f', 'fail_fast', is_flag=True, default=False, help='Stop on failure'), ] def global_test_options(func): for option in reversed(_global_test_options): func = option(func) return func ''' ''' def printd(*args, **kwargs): if os.environ.get('PYROS_DEBUG', '0')=='1': print(*args, **kwargs) ''' GLOBAL_OPTIONS = {} def verbose_mode(): return GLOBAL_OPTIONS["verbose"] def test_mode(): return GLOBAL_OPTIONS["test"] def sim_mode(): return GLOBAL_OPTIONS["sim"] def debug_mode(): return GLOBAL_OPTIONS["debug"] @click.group() @click.option('--debug', '-d', is_flag=True, help='DEBUG mode') @click.option('--sim', '-s', is_flag=True, help="only for an AgentDevice, asking it to start its simulator and work with it (instead of real device)") @click.option('--test', '-t', is_flag=True, help="don't do it for real, just show what it would do") @click.option('--verbose', '-v', is_flag=True, help='Verbose output') # @click.option('--verbose', '-v', 'verbosity', flag_value=2, default=1, help='Verbose output'), # @click.option('--quiet', '-q', 'verbosity', flag_value=0, help='Minimal output'), # @click.option('--fail-fast', '--failfast', '-f', 'fail_fast', is_flag=True, default=False, help='Stop on failure'), def pyros_launcher(debug, sim, test, verbose): # global log # log.info("in pyros launcher") # pass os.environ['PYROS_DEBUG'] = '1' if debug else '0' # pyros_logger.set_logger_config() # log = logging.getLogger('pyroslogger') log.debug('starting pyros') # log.info('info toto msg from pyros') # log.warning('warntoto msg from pyros') # log.error('error toto msg from pyros') # log.critical('error toto msg from pyros') if debug: click.echo('DEBUG mode') if sim: click.echo('Starting and connecting to simulator instead of real device') if test: click.echo('Test mode') if verbose: click.echo('Verbose mode') GLOBAL_OPTIONS["debug"] = debug GLOBAL_OPTIONS["sim"] = sim GLOBAL_OPTIONS["test"] = test GLOBAL_OPTIONS["verbose"] = verbose @pyros_launcher.command(help="Run a pyros shell (django included)") # @global_test_options def shell(): os.environ["PATH_TO_OBSCONF_FILE"] = os.path.join(os.path.abspath( PYROS_DJANGO_BASE_DIR), "obsconfig/fixtures/observatory_configuration_ok_simple.yml") print("Execution commande shell") print("NAME IS", __name__) print() print("Launching a pyros shell") print() print("NB1: If you want to play with an agent, type:") print(" >>> from agent.AgentA import AgentA") print(" >>> agent=AgentA('agent_toto')") print(" >>> agent") print(" >>> agent.run(2) (=> will run 2 iterations)") print(" >>> cmd = agent.send_command('AgentB','eval 2+2')") print(" >>> cmd") print(" >>> cmd.get_updated_result()") print(" >>> ...") print(" - See documentation, section 'Play with a pyros agent' inside the chapter 'Running pyros' for more details") print() print("NB2: If you want to play with the pyros objects, type:") print(" >>> from common.models import *") print(" - (This will import all the pyros objects)") print(" - Then, you can create any pyros object just by typing its name.") print(" - For example, to create an AgentSurvey object, type:") print(" >>> agent_survey = AgentSurvey()") print(" >>> agent_survey") print(" >>> ...") print(" - See documentation, section 'Play with the pyros objects' inside the chapter 'Running pyros' for more details") print() print("Type 'exit()' to quit") print() change_dir(PYROS_DJANGO_BASE_DIR) # execProcess("python install.py install") if not test_mode(): execProcessFromVenv("manage.py shell",foreground=True) # Go back to the initial dir # os.chdir("../") change_dir("PREVIOUS") return True @pyros_launcher.command(help="Run a database (Mysql) shell") def dbshell(): print() print("Launching a database (mysql) shell") print("From this shell, you can type 'show tables;' to see all the pyros tables") print("Then for example, type 'select * from config;' to see the content of the 'config' table") print("Type 'exit' to quit") print() # execProcess("python install.py install") if not test_mode(): execProcessFromVenv(PYROS_DJANGO_BASE_DIR+"/manage.py dbshell") # Go back to the initial dir return True @pyros_launcher.command(help="Update (only if necessary) the python packages AND the source code AND the DB structure") def update(): install_or_update(UPDATE=True) ''' print("Running update command") # 1) Update source code (git pull) printFullTerm(Colors.BLUE, "1) UPDATING SOURCE CODE: Running git pull") _gitpull() or die() # Re-install VENV if disappeared install_venv(False) # 2) Update python packages (pip upgrade AND pip install requirements) printFullTerm(Colors.BLUE, "2) UPDATING PYTHON PACKAGES") # _update_python_packages_from_requirements() or die() install_packages() print(os.getcwd()) # 3) Update PlantUML diagrams printFullTerm(Colors.BLUE, "3) UPDATING PLANTUML DIAGRAMS") _update_plantuml_diags() or die() print(os.getcwd()) # 4) Update database structure (make migrations + migrate) printFullTerm(Colors.BLUE, "4) UPDATING DATABASE") _updatedb() or die() return True ''' @pyros_launcher.command(help="Install the pyros software") @click.option('--packages_only', '-p', is_flag=True, help='install only the python packages (no database installation)') @click.option('--database_only', '-d', is_flag=True, help='install only the pyros database (no python packages installation)') # @global_test_options def install(packages_only, database_only): with_packages = not database_only with_database = not packages_only install_or_update(UPDATE=False, with_packages=with_packages, with_database=with_database) # install_or_update(UPDATE=False, with_packages=packages_only, with_database=database_only) # install_or_update(UPDATE=False, packages_only=packages_only, database_only=database_only) # def install_or_update(UPDATE: bool = False, with_packages: bool = False, with_database: bool = False): def install_or_update(UPDATE: bool = False, with_packages: bool = True, with_database: bool = True): SQL_USER = os.environ.get("MYSQL_PYROS_LOGIN").strip() SQL_PSWD = os.environ.get("MYSQL_PYROS_PWD").strip() os.environ["PATH_TO_OBSCONF_FOLDER"] = os.path.join(os.path.abspath( PYROS_DJANGO_BASE_DIR), "../../../privatedev/config/default/") os.environ["PATH_TO_OBSCONF_FILE"] = os.path.join(os.path.abspath( PYROS_DJANGO_BASE_DIR), os.environ["PATH_TO_OBSCONF_FOLDER"] + "observatory_default.yml") ACTION = "UPDATING" if UPDATE else "INSTALLING" # if WITH_DOCKER: with_database = True if WITH_DOCKER: with_packages = False # if not with_packages and not with_database: with_packages = with_database = True print("- with_packages:", with_packages) print("- with_database:", with_database) # if test_mode(): print("in test mode") # self.execProcess("python3 install/install.py install") # if (os.path.basename(os.getcwd()) != "private"): num = 0 # 1) Update source code (git pull) if UPDATE: num += 1 printFullTerm( Colors.BLUE, f"{num}) UPDATING SOURCE CODE: Running git pull") if not WITH_DOCKER: _gitpull() or die() # 2) Update python packages (pip upgrade AND pip install requirements) if with_packages: num += 1 printFullTerm(Colors.BLUE, f"{num}) {ACTION} PYTHON PACKAGES") # (UPDATE) Re-install VENV if disappeared install_venv(EVEN_IF_ALREADY_EXISTS=not UPDATE) install_packages() if UPDATE: print("Running UPDATE command") else: # Install GitPython package for git support inside python print("Running INSTALL command") try: from git import Repo except: pip = "pip" if IS_WINDOWS else "pip3" if WITH_DOCKER: process = subprocess.Popen( pip + " install --user --upgrade GitPython", shell=True) else: process = subprocess.Popen( pip + " install --upgrade GitPython", shell=True) process.wait() if process.returncode == 0: # self.addExecuted(self.current_command, command) from git import Repo else: log.error("GitPython package (required for obsconfig class) installation failed") ''' print("Guitastro : Generating (updating) API documentation (using Sphinx)") # Make html doc from RST # cd doc/sourcedoc/ change_dir(GUITASTRO_PATH + '/doc_rst/') # ./MAKE_DOC.sh res = execProcess('/bin/bash MAKE_DOC.sh') # Come back to where we were before # cd - change_dir("PREVIOUS") ''' # Guitastro GUITASTRO_PATH = os.path.join(os.getcwd(), "./vendor/guitastro") if with_packages and not WITH_DOCKER : #GUITASTRO_PATH = os.path.join(os.getcwd(), "../vendor/guitastro") change_dir("..") print(os.getcwd()) # 1) clone repo if not yet done if not os.path.exists(GUITASTRO_PATH) and not WITH_DOCKER : print("Guitastro : Cloning repository") cloned_repo = Repo.clone_from( "https://gitlab.irap.omp.eu/guitastrolib/guitastro.git", GUITASTRO_PATH) print("Cloned successfully: ", cloned_repo.__class__ is Repo) if UPDATE and os.path.exists(GUITASTRO_PATH): gitpull_guitastro() change_dir("PREVIOUS") # 2) install/update requirements & generate API doc if os.path.exists(GUITASTRO_PATH) and not WITH_DOCKER: # TODO: update guitastro (git pull from vendor/guitastro/) print("\nGuitastro : Installing/Updating python package dependencies\n") # Upgrade pip if new version available os.system(VENV_PYTHON + ' -m pip install --upgrade pip') # TODO: faire les apt-get intall aussi (indi, ...) venv_pip_install2(GUITASTRO_PATH + '/install/requirements.txt', '-r') #venv_pip_install2(GUITASTRO_PATH + '/install/requirements_linux.txt', '-r') venv_pip_install2(GUITASTRO_PATH + '/install/requirements_dev.txt', '-r') ''' print("Guitastro : Generating (updating) API documentation (using Sphinx)") # Make html doc from RST # cd doc/sourcedoc/ change_dir(GUITASTRO_PATH + '/doc_rst/') # ./MAKE_DOC.sh res = execProcess('/bin/bash MAKE_DOC.sh') # Come back to where we were before # cd - change_dir("PREVIOUS") ''' # 3) Update PlantUML diagrams num += 1 printFullTerm(Colors.BLUE, f"{num}) UPDATING UML DIAGRAMS") _update_plantuml_diags() or die() print(os.getcwd()) # 4) Update Sphinx API doc num += 1 printFullTerm(Colors.BLUE, f"{num}) UPDATING API DOC (with Sphinx)") if not IS_WINDOWS: _update_api_doc() or die() # 5) Install/Update database structure (make migrations + migrate) if with_database: num += 1 printFullTerm(Colors.BLUE, f"{num}) {ACTION} DATABASE") if UPDATE: _updatedb() or die() else: res = install_database(VENV) if(res == 0): print( f"You can connect to PyROS as '{SQL_USER}' user with the password '{SQL_PSWD}' ") return True @pyros_launcher.command(help="Run some tests") @click.option('--app', '-a', help='app name') def test(app): print("Running tests") os.environ["PATH_TO_OBSCONF_FOLDER"] = os.path.join( os.path.abspath(PYROS_DJANGO_BASE_DIR), "obsconfig/fixtures/") os.environ["PATH_TO_OBSCONF_FILE"] = os.path.join(os.path.abspath( PYROS_DJANGO_BASE_DIR), "obsconfig/fixtures/observatory_configuration_ok_simple.yml") os.environ["unit_name"] = "" configfile = 'config_pyros.yml' os.environ["pyros_config_file"] = os.path.join(os.path.abspath( PYROS_DJANGO_BASE_DIR), "../../../config/pyros/", configfile) # start_dir = os.getcwd() if app == None: # apps = ['obsconfig','scientific_program','common', 'scheduler', 'routine_manager', 'user_manager', 'alert_manager.tests.TestStrategyChange'] # Removing alert_manager, scheduler from tests apps = ['obsconfig', "scientific_program", 'common', 'user_manager', 'routine_manager','api'] else: os.environ["PATH_TO_OBSCONF_FILE"] = os.path.join(os.path.abspath( PYROS_DJANGO_BASE_DIR), "obsconfig/fixtures/observatory_configuration_ok_simple.yml") change_dir(PYROS_DJANGO_BASE_DIR) execProcessFromVenv( 'manage.py test --keep --noinput --parallel 4 ' + app,foreground=True) or die() change_dir("PREVIOUS") return True for app in apps: if app != "obsconfig": # for every module except obsconfig, should use the simple observatory configuration in order to run the website with a configuration that isn't the observatory configuration used for production os.environ["PATH_TO_OBSCONF_FILE"] = os.path.join(os.path.abspath( PYROS_DJANGO_BASE_DIR), "obsconfig/fixtures/observatory_configuration_ok_simple.yml") # _loaddata() or die() change_dir(PYROS_DJANGO_BASE_DIR) # Delete test_pyros database after tests # execProcessFromVenv('manage.py test ' + app) or die() # KEEP test_pyros database after tests # execProcessFromVenv('manage.py test --keep ' + app) or die() execProcessFromVenv('manage.py test --keep --noinput ' + app,foreground=True) or die() change_dir("PREVIOUS") # execProcess("python install.py install") return True @pyros_launcher.command(help="Run ALL tests") def testall(): change_dir(PYROS_DJANGO_BASE_DIR) # Delete test_pyros database after tests # execProcessFromVenvAsync("manage.py test") # KEEP test_pyros database after tests execProcessFromVenvAsync("manage.py test --keep") change_dir("PREVIOUS") return True def _gitpull(): print("-- running git pull") GIT = "git.exe" if IS_WINDOWS else "git" if not test_mode(): return execProcess(f"{GIT} pull") return True def gitpull_guitastro(): print("-- running git pull of guitastro") change_dir("./vendor/guitastro/") GIT = "git.exe" if IS_WINDOWS else "git" if not test_mode(): return execProcess(f"{GIT} pull",foreground=True) return True # @pyros_launcher.command(help="Update the pyros database") def _updatedb(): print("-- update db (make migrations + migrate)") if not test_mode(): _makemigrations() or die() _migrate() or die() return True @pyros_launcher.command(help="Update the pyros database and fill it with initial fixture data") def initdb(): if not test_mode(): # updatedb() res1 = _makemigrations() res2 = _migrate() res3 = _loaddata() return res1 == res2 == res3 return True @pyros_launcher.command(help="Launch an agent") # @global_test_options @click.argument('agent') @click.option('--configfile', '-c', help='the configuration file to be used') @click.option('--observatory', '-o', help='the observatory name to be used') @click.option('--unit', '-u', help='the unit name to be used') @click.option("--computer_hostname","-cp", help="The name of simulated computer hostname") @click.option("--foreground","-fg", is_flag=True, help="Print stdout and error in terminal") # @click.option('--format', '-f', type=click.Choice(['html', 'xml', 'text']), default='html', show_default=True) # @click.option('--port', default=8000) # def start(agent:str, configfile:str, test, verbosity): def start(agent: str, configfile: str, observatory: str, unit: str, computer_hostname: str, foreground: bool): log.debug("Running start command") try: from config.pyros.config_pyros import ConfigPyros except: # pip = "pip" if platform.system() == "Windows" else "pip3" # TODO: "python -m pip" au lieu de "pip" pip = "pip" if IS_WINDOWS else "pip3" process = subprocess.Popen( pip + " install --upgrade pykwalify", shell=True) process.wait() if process.returncode == 0: # self.addExecuted(self.current_command, command) from config.pyros.config_pyros import configpyros else: log.error( "pykwalify package (required for obsconfig class) installation failed") if configfile: log.debug("With config file" + configfile) # os.environ["pyros_config_file"] = os.path.join(os.path.abspath( # PYROS_DJANGO_BASE_DIR), "../../../config/pyros/", configfile) os.environ["pyros_config_file"] = os.path.join(os.path.abspath( PYROS_DJANGO_BASE_DIR), "../../../private/config/pyros/", configfile) else: configfile = 'config_pyros.yml' # os.environ["pyros_config_file"] = os.path.join(os.path.abspath( # PYROS_DJANGO_BASE_DIR), "../../../config/pyros/", configfile) os.environ["pyros_config_file"] = os.path.join(os.path.abspath( PYROS_DJANGO_BASE_DIR), "../../../private/config/pyros/", configfile) logo_name = ConfigPyros( os.environ["pyros_config_file"]).pyros_config["general"]["logo"] logo_path = os.path.join(os.path.abspath( PYROS_DJANGO_BASE_DIR), "../../../config/pyros/", logo_name) media_path = os.path.join(os.path.abspath( PYROS_DJANGO_BASE_DIR), "misc/static/media", logo_name) shutil.copy(logo_path, media_path) # if test_mode(): print("in test mode") # if verbose_mode(): print("in verbose mode") if observatory == None or len(observatory) == 0: observatory = "default" observatories_configuration_folder = os.path.join( os.path.abspath(PYROS_DJANGO_BASE_DIR), "../../../privatedev/config/") if len(glob.glob(observatories_configuration_folder+observatory+"/")) != 1: # Observatory configuration folder not found print( f"Observatory configuration folder for observatory '{observatory}' not found in {observatories_configuration_folder}") print("Available observatories configuration :") for obs_conf_folder in os.listdir(observatories_configuration_folder): print(obs_conf_folder) exit(1) path_to_obs_config_folder = observatories_configuration_folder+observatory+"/" obs_config_file_name = "" # Search for observatory config file obs_config_file_name = glob.glob( path_to_obs_config_folder+"/observatory*.yml")[0] obs_config_file_path = os.path.join( path_to_obs_config_folder, obs_config_file_name) os.environ["PATH_TO_OBSCONF_FILE"] = obs_config_file_path os.environ["PATH_TO_OBSCONF_FOLDER"] = path_to_obs_config_folder os.environ["unit_name"] = unit if unit else '' ''' if unit: os.environ["unit_name"] = unit else: os.environ["unit_name"] = "" ''' # add path to pyros_django folder as the config class is supposed to work within this folder # cmd_test_obs_config = f"-c \"from src.core.pyros_django.obsconfig.obsconfig_class import OBSConfig\nOBSConfig('{os.path.join(PYROS_DJANGO_BASE_DIR,os.environ.get('PATH_TO_OBSCONF_FILE'))}')\"" cmd_test_obs_config = f"-c \"from src.core.pyros_django.obsconfig.obsconfig_class import OBSConfig\nOBSConfig('{obs_config_file_path}')\"" if not execProcessFromVenv(cmd_test_obs_config,foreground=True): # Observatory configuration has an issue exit(1) # Check each agent in the given list agents = agent.split(",") if "," in agent else [agent] for a in agents: if not _check_agent(a): return # print("Agents are:", agents) ''' # Check if multiple agents: agents = None if "," in agent: agents = agent.split(",") for a in agents: if not _check_agent(a): return print("Agents are:", agents) # 1 agent only else: agents = [agent] if not _check_agent(agent): return ''' # Start Agents (processes) current_processes = [] for agent_name, agent_folder in AGENTS.items(): # if agent in ("all", agent_name) : if agent == "all" or agent_name in agents: # Default case, launch agentX # if agent_name == "agentX": # execProcessFromVenvAsync(VENV_PYTHON + " manage.py runserver") log.debug(VENV_PYTHON) log.debug("Launching agent" + agent_name + "...") # if not test_mode(): execProcess(VENV_PYTHON + " manage.py runserver") # if not test_mode(): execProcessFromVenv("start_agent_" + agent_name + ".py " + configfile) current_dir = os.getcwd() # OLD format agents: majordome, monitoring, alert... cmd = "start_agent.py " + agent_name + " " + configfile # Agent "webserver" if agent_name == "webserver": cmd = "manage.py runserver 127.0.0.1:8080" if(WITH_DOCKER): # If we're running pyros within docker, we need to specify a specific adress in order to access the website on our host machine PYROS_WEBSITE_PORT = os.environ.get("PYROS_WEBSITE_PORT") cmd = f"manage.py runserver 0.0.0.0:{PYROS_WEBSITE_PORT}" os.chdir(PYROS_DJANGO_BASE_DIR) # if not test_mode(): execProcessFromVenv("start_agent.py " + agent_name + " " + configfile) # Agent "agentM", "agentA", "agentB", "agentX", ... elif agent_name.startswith("agent"): # Run agent without actual commands sent to devices (FOR_REAL=False) # agentX.run(FOR_REAL=True) ''' if agent_name == "agentM": os.chdir(PYROS_DJANGO_BASE_DIR+"/monitoring/") elif agent_name == "agentSP": os.chdir(PYROS_DJANGO_BASE_DIR+"/scientific_program/") elif agent_name == "agentScheduler": os.chdir(PYROS_DJANGO_BASE_DIR+"/scheduler/") else: os.chdir(PYROS_DJANGO_BASE_DIR+"/agent/") ''' #if agent_name in ["agentM", "agentSP", "agentScheduler", "agentImagesProcessor"]: #os.chdir(PYROS_DJANGO_BASE_DIR+"/agent/") os.chdir(PYROS_DJANGO_BASE_DIR + os.sep + agent_folder) # cmd = "-m AgentX" # cmd = f" Agent{agent_name[5:]}.py {configfile}" cmd = f"Agent{agent_name[5:]}.py" if debug_mode(): cmd += " -d" if sim_mode(): cmd += " -s" if test_mode(): cmd += " -t" if verbose_mode(): cmd += " -v" # if configfile: # cmd += f" {configfile}" if computer_hostname: cmd += " -c {computer_hostname}" # if not test_mode(): current_processes.append( [execProcessFromVenvAsync(cmd), agent_name, -1] ) # Append this process ( [process id, agent_name, result=failure] ) # ("result" will be updated at the end of execution) current_processes.append( [execProcessFromVenvAsync(cmd,foreground), agent_name, -1]) # self.change_dir("..") os.chdir(current_dir) # Go back to root folder (/) # self.change_dir('..') # os.chdir("..") # Wait for end of each process execution # for (p,agent) in current_processes: for process in current_processes: p, agent, _ = process log.debug( f"************ Waiting for end of execution of agent {agent} ************") p.wait() process[2] = p.returncode print(f"************ END of execution of agent {agent} ************") if p.returncode == 0: printFullTerm( Colors.GREEN, f"Process {agent} executed successfully") # self.addExecuted(self.current_command, command) else: printFullTerm(Colors.WARNING, f"Process {agent} execution failed") # self.addError(self.current_command, command) print() print() print("Synthesis of the results:") for process in current_processes: p, agent, returncode = process if returncode == 0: printFullTerm( Colors.GREEN, f"Process {agent} executed successfully") # self.addExecuted(self.current_command, command) else: printFullTerm(Colors.WARNING, f"Process {agent} execution failed") # self.addError(self.current_command, command) # print("************ end of START() ************") # Only according to the last process status: # return True if p.returncode==0 else False return True if p.returncode == 0 else False @pyros_launcher.command(help="Launch agentSST") # @global_test_options @click.option('--configfile', '-c', help='the configuration file to be used') @click.option('--observatory', '-o', help='the observatory name to be used') @click.option('--unit', '-u', help='the unit name to be used') @click.option("--computer_hostname","-cp", help="The name of simulated computer hostname") @click.option("--agent","-a", help="The name of agent to be launched") @click.option("--foreground","-fg", is_flag=True, help="Print stdout and error in terminal") def new_start(configfile: str, observatory: str, unit: str, computer_hostname: str,foreground:bool,agent:str): log.debug("Running start command") try: from config.pyros.config_pyros import ConfigPyros except: # pip = "pip" if platform.system() == "Windows" else "pip3" # TODO: "python -m pip" au lieu de "pip" pip = "pip" if IS_WINDOWS else "pip3" process = subprocess.Popen( pip + " install --user --upgrade pykwalify", shell=True) process.wait() if process.returncode == 0: # self.addExecuted(self.current_command, command) from config.pyros.config_pyros import configpyros else: log.error( "pykwalify package (required for obsconfig class) installation failed") if configfile: log.debug("With config file" + configfile) os.environ["pyros_config_file"] = os.path.join(os.path.abspath( PYROS_DJANGO_BASE_DIR), "../../../config/pyros/", configfile) else: configfile = 'config_pyros.yml' os.environ["pyros_config_file"] = os.path.join(os.path.abspath( PYROS_DJANGO_BASE_DIR), "../../../config/pyros/", configfile) logo_name = ConfigPyros( os.environ["pyros_config_file"]).pyros_config["general"]["logo"] logo_path = os.path.join(os.path.abspath( PYROS_DJANGO_BASE_DIR), "../../../config/pyros/", logo_name) media_path = os.path.join(os.path.abspath( PYROS_DJANGO_BASE_DIR), "misc/static/media", logo_name) shutil.copy(logo_path, media_path) # if test_mode(): print("in test mode") # if verbose_mode(): print("in verbose mode") if observatory == None or len(observatory) == 0: observatory = "default" observatories_configuration_folder = os.path.join( os.path.abspath(PYROS_DJANGO_BASE_DIR), "../../../privatedev/config/") if len(glob.glob(observatories_configuration_folder+observatory+"/")) != 1: # Observatory configuration folder not found print( f"Observatory configuration folder for observatory '{observatory}' not found in {observatories_configuration_folder}") print("Available observatories configuration :") for obs_conf_folder in os.listdir(observatories_configuration_folder): print(obs_conf_folder) exit(1) path_to_obs_config_folder = observatories_configuration_folder+observatory+"/" obs_config_file_name = "" # Search for observatory config file obs_config_file_name = glob.glob( path_to_obs_config_folder+"/observatory*.yml")[0] obs_config_file_path = os.path.join( path_to_obs_config_folder, obs_config_file_name) os.environ["PATH_TO_OBSCONF_FILE"] = obs_config_file_path os.environ["PATH_TO_OBSCONF_FOLDER"] = path_to_obs_config_folder os.environ["unit_name"] = unit if unit else '' ''' if unit: os.environ["unit_name"] = unit else: os.environ["unit_name"] = "" ''' # add path to pyros_django folder as the config class is supposed to work within this folder # cmd_test_obs_config = f"-c \"from src.core.pyros_django.obsconfig.obsconfig_class import OBSConfig\nOBSConfig('{os.path.join(PYROS_DJANGO_BASE_DIR,os.environ.get('PATH_TO_OBSCONF_FILE'))}')\"" cmd_test_obs_config = f"-c \"from src.core.pyros_django.obsconfig.obsconfig_class import OBSConfig\nOBSConfig('{obs_config_file_path}')\"" if not execProcessFromVenv(cmd_test_obs_config,foreground=True): # Observatory configuration has an issue exit(1) current_processes = [] # cmd = "manage.py runserver" # if(WITH_DOCKER): # # If we're running pyros within docker, we need to specify a specific adress in order to access the website on our host machine # PYROS_WEBSITE_PORT = os.environ.get("PYROS_WEBSITE_PORT") # cmd = f"manage.py runserver 0.0.0.0:{PYROS_WEBSITE_PORT}" # os.chdir(PYROS_DJANGO_BASE_DIR) # current_processes.append( # [execProcessFromVenvAsync(cmd), "webserver", -1]) # Start AgentSST (process) agent_name = "agentSST" agent_folder = AGENTS.get(agent_name) current_dir = os.getcwd() os.chdir(PYROS_DJANGO_BASE_DIR + os.sep + agent_folder) cmd = f"Agent{agent_name[5:]}.py" if debug_mode(): cmd += " -d" if sim_mode(): cmd += " -s" if test_mode(): cmd += " -t" if verbose_mode(): cmd += " -v" if computer_hostname: cmd += f" --computer {computer_hostname}" if agent: cmd += f" --agent {agent}" # if not test_mode(): current_processes.append( [execProcessFromVenvAsync(cmd), agent_name, -1] ) # Append this process ( [process id, agent_name, result=failure] ) # ("result" will be updated at the end of execution) process = execProcessFromVenvAsync(cmd,foreground) # self.change_dir("..") os.chdir(current_dir) # Go back to root folder (/) # self.change_dir('..') # os.chdir("..") # Wait for end of each process execution # for (p,agent) in current_processes: agent = agent_name log.debug( f"************ Waiting for end of execution of agent {agent} ************") try: process.wait() except KeyboardInterrupt: process.send_signal(signal.SIGINT) process.wait() print(f"************ END of execution of agent {agent} ************") if process.returncode == 0: printFullTerm( Colors.GREEN, f"Process {agent} executed successfully") # self.addExecuted(self.current_command, command) else: printFullTerm(Colors.WARNING, f"Process {agent} execution failed") # self.addError(self.current_command, command) return True if process.returncode == 0 else False @pyros_launcher.command(help="Stop pyros") def stop(): print("Running stop command") try: for line in os.popen("ps ax | grep " + "AgentSST" + " | grep -v grep"): fields = line.split() pid = fields[0] # terminating process os.kill(int(pid), signal.SIGINT) except: print("Error Encountered while try to kill agentSST") """ ******************************************************************************** ******************** PRIVATE FUNCTIONS DEFINITION ****************************** ******************************************************************************** """ def notused_update_python_packages_from_requirements(): # 1) Upgrade pip (if new version available) res = execProcessFromVenv("-m pip install --user --upgrade pip") if not res: return False # 2) Install only "not yet installed" python packages res = execProcessFromVenv("-m pip install --user -r install/"+REQUIREMENTS) return res def _update_plantuml_diags(): for dirpath, dirnames, files in os.walk(PYROS_DJANGO_BASE_DIR): if os.path.basename(dirpath) == "doc": diagrams = glob.glob(dirpath+os.sep+"*.pu") # For each diagram source file (.pu), generate the diagram PNG file (only if CHANGED) for diag in diagrams: # if no diag.png or if diag.pu more recent than diag.png => re-generate diag.png diag_png = diag.split('.pu')[0] + '.png' if not os.path.isfile(diag_png) or (os.path.getmtime(diag) > os.path.getmtime(diag_png)): # res = execProcessFromVenv("-m plantuml "+diag) # we need to use our own version of plantuml res = execProcessFromVenv("./install/plantuml.py "+diag,foreground=True) if not res: return False return True def _update_api_doc(): print(os.getcwd()) DOC_RST_PATH = "doc/doc_rst/" # 0) Upgrade pip if new version available if WITH_DOCKER: os.system(VENV_PYTHON + ' -m pip install --user --upgrade pip') else: os.system(VENV_PYTHON + ' -m pip install --upgrade pip') # 1) install/update Sphinx requirements (only if not yet installed) venv_pip_install2(DOC_RST_PATH+'requirements.txt', '-r') # 2) make html doc from RST doc # cd doc/sourcedoc/ change_dir(DOC_RST_PATH) # ./make_rst_then_html res = execProcess('make clean ; /bin/bash make_rst_then_html') print("The technical (and API) documentation has been generated and can be opened in a local browser with this command :") print(" open "+DOC_RST_PATH+"build/html/index.html") print(" (or just open your local browser _firefox, chrome, ..._ and go to this file)") # Come back to where we were before # cd - change_dir("PREVIOUS") return True def _migrate(): change_dir(PYROS_DJANGO_BASE_DIR) # Migrate only migrations for the app "common" # res = execProcessFromVenv("manage.py migrate common") # Migrate all migrations for ALL apps res = execProcessFromVenv("manage.py migrate",foreground=True) change_dir("PREVIOUS") return res def _makemigrations(): change_dir(PYROS_DJANGO_BASE_DIR) # execProcessFromVenv(self.venv_bin + " manage.py makemigrations") # res = execProcessFromVenv("manage.py makemigrations common") res = execProcessFromVenv("manage.py makemigrations",foreground=True) change_dir("PREVIOUS") return res # TODO: mettre la fixture en date naive (sans time zone) def _loaddata(): change_dir(PYROS_DJANGO_BASE_DIR) # execProcessFromVenv(self.venv_bin + " manage.py loaddata misc" + os.sep + "fixtures" + os.sep + self.INIT_FIXTURE) res = execProcessFromVenv( "manage.py loaddata misc" + os.sep + "fixtures" + os.sep + INIT_FIXTURE) change_dir("PREVIOUS") return res def change_dir(path): global _previous_dir if path == "PREVIOUS": path = _previous_dir _previous_dir = os.getcwd() log.debug("Moving to : " + path) os.chdir(path) log.debug("Current directory : " + str(os.getcwd())) def _check_agent(agent): # Check that agent exists if agent != "all" and agent not in AGENTS.keys(): # Check if multiple agents: print("This agent does not exist") print("Here is the allowed list of agents:") print("- all => will launch ALL agents") for agent_name in AGENTS.keys(): print('-', agent_name) return False return True """ INSTALLATION FUNCTIONS (./pyros.py install) """ def replacePatternInFile(pattern, replace, file_path): try: with fileinput.FileInput(file_path, inplace=True, backup='.bak') as file: for line in file: print(line.replace(pattern, replace), end='') except: sys.stderr.write( Colors.ERROR + "ERROR !: replacement in file failed !" + Colors.END + "\r\n") return 1 return 0 def notused_install_dependency_ubuntu(command, mode): ''' Install dependency then check the return code ''' old = command if (mode == 'i'): command = 'apt-get install ' + command elif (mode == 'u'): command = 'apt-get update' elif (mode == 'a'): command = 'add-apt-repository ' + command process = subprocess.Popen(command, shell=True) process.wait() if process.returncode != 0: sys.stderr.write(Colors.ERROR + "ERROR !: installation of " + old + " failed !" + Colors.END + "\r\n") def notused_install_required_ubuntu(): install_dependency_ubuntu("update", 'u') install_dependency_ubuntu("python-lxml", 'i') install_dependency_ubuntu("libxml2-dev", 'i') install_dependency_ubuntu("libxslt-dev", 'i') install_dependency_ubuntu("zlib1g-dev", 'i') install_dependency_ubuntu("update", 'u') install_dependency_ubuntu("rabbitmq-server", 'i') # install_dependency_ubuntu("libmysqlclient-dev", 'i') def notused_install_dependency_centos(command, mode): old = command if (mode == 'i'): command = 'yum -y install ' + command elif (mode == 'u'): command = 'yum update ' + command process = subprocess.Popen(command, shell=True) process.wait() if process.returncode != 0: sys.stderr.write(Colors.ERROR + "ERROR !: installation of " + old + " failed !" + Colors.END + "\r\n") def notused_install_required_centos(): install_dependency_centos("yum", 'u') install_dependency_centos("kernel", 'u') install_dependency_centos("", 'u') install_dependency_centos("libxml2", 'i') install_dependency_centos("libxslt libxslt-2", 'i') install_dependency_centos("libxslt-devel libxml2-devel", 'i') install_dependency_centos("rabbitmq-server", 'i') install_dependency_centos("mariadb-server", 'i') install_dependency_centos("mariadb", 'i') install_dependency_centos("mariadb-devel", 'i') process = subprocess.Popen("systemctl start mariadb.service", shell=True) process.wait() if process.returncode != 0: sys.stderr.write(Colors.ERROR + "ERROR !" + Colors.END + "\r\n") process = subprocess.Popen("systemctl enable mariadb.service", shell=True) process.wait() if process.returncode != 0: sys.stderr.write(Colors.ERROR + "ERROR !" + Colors.END + "\r\n") process = subprocess.Popen("mysql_secure_installation", shell=True) process.wait() if process.returncode != 0: sys.stderr.write(Colors.ERROR + "ERROR !" + Colors.END + "\r\n") def notused_install_required(): # Checking if user is sudo then install the needed dependencies # Find the linux distribution and call the related function distribution = platform.dist() if not 'SUDO_UID' in os.environ.keys(): sys.stderr.write( "Super user rights are needed to install prerequisites\r\n") exit(1) if distribution[0] == "Ubuntu" or distribution[0] == "Debian": install_required_ubuntu() elif distribution[0] == "centos": install_required_centos() else: print("Requirements are made for Ubuntu, Debian and CentOS only") exit(1) def venv_pip_install(package_name: str, options: str = ''): if WITH_DOCKER: os.system(VENV_PIP + ' install --user ' + options + ' ' + package_name) else: os.system(VENV_PIP + ' install ' + options + ' ' + package_name) def venv_pip_install2(package_name: str, options: str = ''): if WITH_DOCKER: os.system(VENV_PYTHON + ' -m pip install --user ' + options + ' ' + package_name) else: os.system(VENV_PYTHON + ' -m pip install ' + options + ' ' + package_name) def install_venv(EVEN_IF_ALREADY_EXISTS: bool = False): ''' Install VENV if does not exist OR Re-install it if EVEN_IF_ALREADY_EXISTS is true (return true if no issue) ''' # -------------------------------------------- # --- Be aware not to create virtual environment in case of user root # -------------------------------------------- if 'SUDO_UID' in os.environ.keys(): answer = input( "You are about to install your virtualenv only for root, this is discouraged, are you sure ? (Y/N) If you are not sure, relaunch the script without super user privileges\n") while (answer != 'Y' and answer != 'y' and answer != 'n' and answer != 'N'): answer = input( "You are about to install your virtualenv only for root, this is discouraged, are you sure ? (Y/N) \n") if (answer not in ['y', 'Y']): exit(1) # -------------------------------------------- # --- (If not exists) Create the (private) venv ROOT directory (where the virtual environment dir will be created) # -------------------------------------------- if (os.path.basename(os.getcwd()) != VENV_ROOT): # if not(os.path.isdir("../venv")): if not(os.path.isdir(VENV_ROOT)): print(Colors.LOG_BLUE + f"-----------------------------Creating 'venv' root directory ({VENV_ROOT}/)-----------------------------" + Colors.END) # os.mkdir("../venv") os.mkdir(VENV_ROOT) os.chdir(VENV_ROOT) # -------------------------------------------- # --- (if EVEN_IF_ALREADY_EXISTS) Delete venv dir if already exist # -------------------------------------------- # print(Colors.LOG_BLUE + "-----------------------------cd venv-----------------------------" + Colors.END) if EVEN_IF_ALREADY_EXISTS: while True: try: if os.path.isdir(VENV): print( Colors.LOG_BLUE + f"-----------------------------Deleting existing VENV dir ({VENV}/)-----------------------------" + Colors.END) shutil.rmtree(VENV) break # Exception on Windows WinError 145 : Cannot remove folder because files in folder not yet removed... except Exception as e: # print(e) continue # -------------------------------------------- # --- (if not exists) Reinstall the virtual environment (from ../venv/) # -------------------------------------------- if not os.path.isdir(VENV): print(Colors.LOG_BLUE + f"-----------------------------Creating venv dir ({VENV}/)-----------------------------"+END_OF_LINE + Colors.END) os.system(GLOBAL_PYTHON+" -m venv " + VENV) # Come back to project root os.chdir('..') return True def install_packages(): os.chdir(VENV_ROOT) print(Colors.LOG_BLUE + "-----------------------------Upgrade pip, wheel, and setuptools" + "-----------------------------"+END_OF_LINE + Colors.END) # Upgrade pip if WITH_DOCKER: os.system(VENV_PYTHON + ' -m pip install --user --upgrade pip') else: os.system(VENV_PYTHON + ' -m pip install --upgrade pip') ''' if (platform.system() == "Windows"): os.system(venv + '\Scripts\python -m pip install --upgrade pip') else: # Linux os.system(venv + '/bin/python -m pip install --upgrade pip') ''' # Pip upgrade wheel and setuptools venv_pip_install('wheel', '--upgrade') # os.system(VENV_PIP+' install --upgrade wheel') # Not working with python 3.8 (on 17/02/2022) # venv_pip_install('setuptools', '--upgrade') if WITH_DOCKER: os.system(VENV_PIP+' install --user setuptools==58') else: os.system(VENV_PIP+' install setuptools==58') # Pip install required packages from REQUIREMENTS file print() # General normal packages print(Colors.LOG_BLUE + "-----------------------------Installing python packages via pip-----------------------------" + Colors.END) venv_pip_install('../install/'+REQUIREMENTS, '-r') # os.system(VENV_PIP+' install -r ../install' + os.sep + REQUIREMENTS) # DEV only packages if DEV: print(Colors.LOG_BLUE + "-----------------------------Installing DEV python packages via pip-----------------------------" + Colors.END) venv_pip_install('../install/'+REQUIREMENTS_DEV, '-r') print("FIN INSTALL PACKAGES") return None # print(Colors.LOG_BLUE + "-----------------------------cd ../install-----------------------------" + Colors.END) if IS_WINDOWS: os.chdir("../install") # moving voeventparse in site-packages directory try: site_packages = "..\\venv\\"+VENV+"\\Lib\\site-packages\\" if ( not os.path.isdir( site_packages + "voevent_parse-0.9.5.dist-info") and not os.path.isdir(site_packages + "voeventparse") ): print(Colors.LOG_BLUE + "\r\n\r\n-----------------------------Copying the voevent library in Lib/site-packages-----------------------------" + Colors.END) cmdline = "xcopy /i /y windows\\voeventparse " + site_packages + "voeventparse" process = subprocess.Popen(cmdline) process.wait() if (process.returncode != 0): raise Exception process = subprocess.Popen( "xcopy /i /y windows\\voevent_parse-0.9.5.dist-info " + site_packages + "voevent_parse-0.9.5.dist-info") process.wait() if (process.returncode != 0): raise Exception print(Colors.LOG_BLUE + "\r\n-----------------------------library successfully copied-----------------------------" + Colors.END) except Exception as e: # , file=stderr) print( Colors.ERROR + "ERROR while Copying the voevent library in Lib/site-packages" + Colors.END) return False # Go back to project root dir os.chdir('..') # return 0 return True def install_database(venv): SQL_USER = os.environ.get("MYSQL_PYROS_LOGIN").strip() SQL_PSWD = os.environ.get("MYSQL_PYROS_PWD").strip() print(Colors.LOG_BLUE + END_OF_LINE + "-----------------------------Launching mysql to create database and create and grant user pyros-----------------------------" + Colors.END) # -------------------------------------------- # --- Determine the MySQL version # -------------------------------------------- output = subprocess.check_output("mysql --version", shell=True) # output is something like: "mysql Ver 15.1 Distrib 10.0.20-MariaDB, for Linux (x86_64) using EditLine wrapper" tmp = (str(output).split()[2]).split('.') sql_version = float(tmp[0]+'.'+tmp[1]) print(Colors.LOG_BLUE + "MySQL version is " + str(sql_version) + Colors.END) # -------------------------------------------- # --- Prepare the SQL query to create and initialize the pyros database if needed # -------------------------------------------- ''' (OLD) sql_query = \ f"CREATE DATABASE IF NOT EXISTS {SQL_DATABASE}; " +\ f"CREATE DATABASE IF NOT EXISTS {SQL_DATABASE_SIMU}; " if sql_version < 5.5: # sql_query = "drop database "+SQL_DATABASE+" ; CREATE DATABASE "+SQL_DATABASE+"; drop database "+SQL_DATABASE_SIMU+" ; CREATE DATABASE "+SQL_DATABASE_SIMU+"; CREATE USER "+SQL_USER+" ; GRANT USAGE ON *.* TO '"+SQL_USER+"'@'localhost' IDENTIFIED BY '"+SQL_PSWD+"' WITH GRANT OPTION; DROP USER '"+SQL_USER+"'@'localhost'; GRANT ALL ON "+SQL_DATABASE+".* TO '"+SQL_USER+"'@'localhost' IDENTIFIED BY '"+SQL_PSWD+"'; GRANT ALL PRIVILEGES ON "+SQL_DATABASE+".* TO '"+SQL_USER+"'@'localhost' IDENTIFIED BY '"+SQL_PSWD+"' WITH GRANT OPTION; GRANT ALL PRIVILEGES ON "+SQL_DATABASE_SIMU+".* TO "+SQL_USER+"@localhost IDENTIFIED BY '"+SQL_PSWD+"' WITH GRANT OPTION;" sql_query = \ f"DROP DATABASE {SQL_DATABASE}; CREATE DATABASE {SQL_DATABASE}; " +\ f"DROP DATABASE {SQL_DATABASE_SIMU}; CREATE DATABASE {SQL_DATABASE_SIMU}; " sql_query += "CREATE USER "+SQL_USER+" ; GRANT USAGE ON *.* TO '"+SQL_USER+"'@'localhost' IDENTIFIED BY '"+SQL_PSWD+"' WITH GRANT OPTION; " sql_query += "DROP USER '"+SQL_USER+"'@'localhost'; " sql_query += "GRANT ALL ON "+SQL_DATABASE+".* TO '"+SQL_USER+"'@'localhost' IDENTIFIED BY '"+SQL_PSWD+"'; " sql_query += "GRANT ALL PRIVILEGES ON "+SQL_DATABASE+".* TO '"+SQL_USER+"'@'localhost' IDENTIFIED BY '"+SQL_PSWD+"' WITH GRANT OPTION; " sql_query += "GRANT ALL PRIVILEGES ON "+SQL_DATABASE_SIMU+".* TO "+SQL_USER+"@localhost IDENTIFIED BY '"+SQL_PSWD+"' WITH GRANT OPTION; " ''' # 1) Create databases pyros and pyros_test (but not test_pyros because will be automatically created by django) # TODO: Pour mysql < 5.5, comment éviter un "drop database" inutile (si la BD n'existe pas encore) qui va provoquer un plantage mysql ? IF_EXISTS = '' if sql_version < 5.5 else 'IF EXISTS' sql_query = f"DROP DATABASE {IF_EXISTS} {SQL_DATABASE}; CREATE DATABASE {SQL_DATABASE};" sql_query += f"DROP DATABASE {IF_EXISTS} {SQL_DATABASE_SIMU}; CREATE DATABASE {SQL_DATABASE_SIMU};" sql_query += f"DROP DATABASE {IF_EXISTS} {SQL_DATABASE_TEST}; CREATE DATABASE {SQL_DATABASE_TEST};" # 2) Create user pyros and give it all rights on 3 databases # Ne marche pas si l'utilisateur existe déjà => erreur # "CREATE USER "+SQL_USER+"; " +\ # Donc, il faut ruser # Mysql >= 5.7 only: # sql_query_elem = "CREATE USER IF NOT EXISTS "+SQL_USER+"; " # Si user n'existe pas => est créé ; Si user existe => pas d'erreur ; DONC ok dans les 2 cas # sql_query_elem = "GRANT ALL ON "+SQL_DATABASE+".* TO '"+SQL_USER+"'@'localhost' IDENTIFIED BY '"+SQL_PSWD+"'; " if WITH_DOCKER: # we will use this later (at the end of development) # host = os.environ["IP_PYROS_USER"] host = "%" else: host = "localhost" # FOR MYSQL 8: if sql_version >= 8: # We remove identified by because grant was creating the pyros user, wasn't working on version 8 so we separated the steps : # first we create the user then we grant him rights sql_query += f"CREATE USER IF NOT EXISTS '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; " sql_query += f"GRANT ALL ON {SQL_DATABASE}.* TO '{SQL_USER}'@'{host}'; " sql_query += f"GRANT ALL ON {SQL_DATABASE_SIMU}.* TO '{SQL_USER}'@'{host}'; " sql_query += f"GRANT ALL ON {SQL_DATABASE_TEST}.* TO '{SQL_USER}'@'{host}'; " else: sql_query += f"GRANT ALL ON {SQL_DATABASE}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; " sql_query += f"GRANT ALL ON {SQL_DATABASE_SIMU}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; " # This database does not yet exists and will be automatically created by Django, but we already give access rights to it for pyros user sql_query += f"GRANT ALL ON {SQL_DATABASE_TEST}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; " # FOR MYSQL 8: if sql_version >= 8: sql_query += "FLUSH PRIVILEGES; " # "GRANT USAGE ON *.* TO '"+SQL_USER+"'@localhost ; " ''' # (EP) AVANT, y avait tout ça..., vraiment utile ? "GRANT USAGE ON *.* TO '"+SQL_USER+"'; " +\ "DROP USER '"+SQL_USER+"'; " +\ "GRANT ALL ON "+SQL_DATABASE+".* TO '"+SQL_USER+"'@'localhost' IDENTIFIED BY '"+SQL_PSWD+"'; " +\ "GRANT ALL ON "+SQL_DATABASE_SIMU+".* TO '"+SQL_USER+"'@'localhost'; " +\ "GRANT ALL PRIVILEGES ON "+SQL_DATABASE_SIMU+".* TO '"+SQL_USER+"'@'localhost'; " +\ "GRANT ALL ON "+SQL_DATABASE_SIMU+".* TO '"+SQL_USER+"'@'localhost' IDENTIFIED BY '"+SQL_PSWD+"' ;" ''' # NEWER MYSQL: # OLDER MYSQL: Try this instead for OLDER mysql (works on CentOS 6.4 and Centos 7.5 with mysql 5.5): # req = "drop database pyros; CREATE DATABASE pyros; drop database pyros_test ; CREATE DATABASE pyros_test; drop user 'pyros'@'localhost' ; CREATE USER pyros; GRANT USAGE ON *.* TO 'pyros'; DROP USER 'pyros'; GRANT ALL ON pyros.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros'; GRANT ALL ON test_pyros.* TO 'pyros'@'localhost'; GRANT ALL PRIVILEGES ON test_pyros_test.* TO 'pyros'@'localhost'; GRANT ALL ON pyros_test.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros'" # req = "drop database pyros ; CREATE DATABASE pyros; drop database pyros_test ; CREATE DATABASE pyros_test; DROP USER 'pyros'@'localhost' ; GRANT USAGE ON *.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros' WITH GRANT OPTION; DROP USER 'pyros'@'localhost'; GRANT ALL ON pyros.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros'; GRANT ALL PRIVILEGES ON pyros.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros' WITH GRANT OPTION; GRANT ALL PRIVILEGES ON pyros_test.* TO pyros@localhost IDENTIFIED BY 'DjangoPyros' WITH GRANT OPTION;" # (EP) ok for CENTOS 7 I suppose (but not for CentOS 6): # req_centos = "CREATE DATABASE IF NOT EXISTS pyros; CREATE DATABASE IF NOT EXISTS pyros_test; GRANT USAGE ON *.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros' WITH GRANT OPTION; DROP USER 'pyros'@'localhost'; GRANT ALL ON pyros.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros'; GRANT ALL PRIVILEGES ON pyros.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros' WITH GRANT OPTION; GRANT ALL PRIVILEGES ON pyros_test.* TO pyros@localhost IDENTIFIED BY 'DjangoPyros' WITH GRANT OPTION;" # --- Prepare the SQL query to create and initialize the pyros database if needed # if platform.dist()[0] == "centos": # req = sql_query # mysql_call_root = '"' + MYSQL_EXE_PATH + 'mysql" -u root -p' # if the mysql_root_password isn't defined, it will ask to enter the password try: # a root password should be defined as an environment variable mysql_root_password = os.environ['MYSQL_ROOT_PASSWORD'].strip() mysql_login = os.environ['MYSQL_ROOT_LOGIN'].strip() except: # need a space as string so the command ask to the user to write his password. mysql_root_password = " " mysql_login = "root" if WITH_DOCKER: # host is the name of the service of database in docker-compose (i.e. host=db) mysql_host = "db" mysql_login = "root" else: mysql_host = "localhost" # when using -p option, there is no space between the option and the password value mysql_call_root = f"mysql -h {mysql_host} -u '{mysql_login}' -p'{mysql_root_password}'" mysql_call_pyros = "\"" + MYSQL_EXE_PATH + "mysql\" -u "+SQL_USER+" -p" # -------------------------------------------- # --- Creating database and creating and granting user pyros # -------------------------------------------- user_ros_is_created = True if sql_version < 5.5: print(Colors.LOG_BLUE + "------------------ Check if the user pyros exists in MYSQL (type the pyros password) -----------------------------" + Colors.END) # --- We are testing if user pyros already exists in the database process = subprocess.Popen( "echo quit |" + mysql_call_pyros, shell=True) process.wait() if (process.returncode == 0): user_ros_is_created = False if user_ros_is_created: # --- The user pyros must be created in the database print(Colors.LOG_BLUE + "-----------------------------Please enter your MYSQL root password-----------------------------" + Colors.END) # process = subprocess.Popen("echo \"" + sql_query + "\" |"+ mysql_call_root, shell=True) if IS_WINDOWS: mysql_call_root = f"mysql -h {mysql_host} -u '{mysql_login}' -p{mysql_root_password}" sql_cmd = 'echo ' + sql_query + ' | ' + mysql_call_root else: sql_cmd = 'echo "' + sql_query + '" | ' + mysql_call_root print("Executing sql cmd: ", sql_cmd) process = subprocess.Popen(sql_cmd, shell=True) process.wait() if (process.returncode != 0): sys.stderr.write( Colors.ERROR + "ERROR !: db configuration failed !" + Colors.END + "\r\n") return -1 print(Colors.LOG_BLUE + END_OF_LINE + "-----------------------------Database created and user pyros successfully created and granted-----------------------------" + Colors.END) # -------------------------------------------- # --- Replacing pattern in settings.py to use mysql # -------------------------------------------- print(Colors.LOG_BLUE + "-----------------------------setting MYSQL = True in settings-----------------------------" + Colors.END) replacePatternInFile("MYSQL = False", "MYSQL = True", os.path.normpath( f"{PYROS_DJANGO_BASE_DIR}/pyros/settings.py")) # print(Colors.LOG_BLUE + "\r\n-----------------------------cd ..-----------------------------" + Colors.END) # 2021 : no need to change folder in order to init the database (maybe the next line was forget to be commented because the previous line is commented) # os.chdir("..") # -------------------------------------------- # --- Executing migrations # -------------------------------------------- print(Colors.LOG_BLUE + "\r\n\r\n-----------------------------Migrate : executing pyros.py init_database-----------------------------" + Colors.END) # TODO: from venv !!! try: # os.system(GLOBAL_PYTHON+" pyros.py init_database") res = execProcessFromVenv("pyros.py initdb") #res = os.system(GLOBAL_PYTHON+" pyros.py initdb") return res ''' process = subprocess.Popen(GLOBAL_PYTHON + " pyros.py init_database" , shell=True) process.wait() ''' except Exception as e: print("Exception ", e) print(Colors.ERROR + "Error while initializing database :" + Colors.END) return -1 print(Colors.LOG_BLUE + "\r\n\r\n-----------------------------Install successfull !-----------------------------" + Colors.END) return 0 def notused_help(): print( # "Welcome in the installation script of the pyros venv.\t\nPlease launch it from the install directory of pyros.\n\tIf you're on Ubuntu Debian or CentOS:\n\tlaunch it with sudo and <--prerequisites> or <-p> to install the prerequisites.\n\t-->sudo ./test_install.py -p\n\n\tFor the python packages launch it from the install directory of pyros without sudo and without parameter\n\t-->./test_install.py") "Welcome to the installation script of the pyros software.\t\n" + \ "Usage:\n" + \ " [python] ./pyros.py install [-p] [-d]" ) """ ******************************************************************************** ********************************* main() FUNCTION ****************************** ******************************************************************************** """ def main(): set_environment_variables_if_not_configured(ENV_PATH, ENV_SAMPLE_PATH) pyros_launcher() # AVIRER # @click.command() # @click.argument('start') def oldmain(): ''' cli() return ''' # fire.Fire(Commands) # return # if len(sys.argv) == 3 and sys.argv[1].startswith("simulator"): SIMULATOR_CONFIG_FILE = sys.argv[2] # print(sys.argv) """ system = platform.system() columns = 100 row = 1000 disp = True """ # Read command if len(sys.argv) <= 1: die("You must give a command name") command = sys.argv[1] if not command in COMMANDS.keys(): die("This command does not exist") command_all_args = COMMANDS[command] # Read command args if should be command_arg = None if command_all_args: if len(sys.argv) <= 2: die("This command should be given an argument") command_arg = sys.argv[2] if not command_arg in command_all_args: die("This argument does not exist for command " + command) print("Executing command", command) if command_arg: print("with arg", command_arg) # command(command_arg) if command_arg: globals()[command](command_arg) else: globals()[command]() # sys.exit(pyros.exec()) if __name__ == "__main__": main() """ INSTALLATION main function if __name__ == '__main__': if len(sys.argv) > 2: _help() ; sys.exit(1) # print(sys.argv) if len(sys.argv) == 2: INSTALL_VENV = INSTALL_DB = False if sys.argv[1] == '-p': INSTALL_VENV=True if sys.argv[1] == '-d': INSTALL_DB=True ''' # Prerequisistes installation (for CentOS) => not a good idea for now, deactivated (EP) if sys.argv[1] == "--prerequisites" or sys.argv[1] == "-p": install_required() else: _help() ''' if INSTALL_VENV: install_venv(VENV) if INSTALL_DB: install_database(VENV) """