From e4c37a1aecdea2e501a71e78a170b2254bc5110f Mon Sep 17 00:00:00 2001 From: Alexis Koralewski Date: Fri, 18 Feb 2022 15:50:03 +0100 Subject: [PATCH] Upgrading mysql version and fixing sql queries --- CHANGELOG | 10 ++++++++++ Dockerfile | 5 +++-- VERSION | 2 +- docker/PYROS_DOCKER_BUILD.bat | 2 +- docker/PYROS_DOCKER_INSTALL | 2 ++ docker/PYROS_DOCKER_START.bat | 2 +- docker/docker-compose.yml | 18 +++++++++++------- pyros.py | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++----------------- 8 files changed, 82 insertions(+), 29 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index b13e970..2edb0b6 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,13 @@ +18-02-2022 (AKo): v0.3.7.0 + - Upgrade Mysql version + - Fix sql queries for mysql 8.0 + - Add pyros_user in sequence list and detail + - Add description field in observatories configuration + - Add those description as tooltips when they are displayed sequence fields + - Add local fontawesome css + - Rename configpyros properly (ConfigPyros) + - Fix dockerfile issue with permission (uid needs to be the same as the host) + 08-02-2022 (AKo) : v0.3.6.0 - Add DRF (Django Rest Framework) to requirements - Add Pyros API (User and sequences with sequence submission) diff --git a/Dockerfile b/Dockerfile index 8cf1af4..5473fa2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ FROM python:3.8 +ARG uid RUN apt-get update && apt-get install -y \ default-mysql-client \ @@ -9,9 +10,9 @@ RUN apt-get update && apt-get install -y \ # Get IRAP self signed certificate RUN echo | openssl s_client -connect gitlab.irap.omp.eu:443 -servername gitlab.irap.omp.eu 2>/dev/null | openssl x509 > /etc/ssl/certs/gitlab.irap.omp.eu.crt - +RUN echo "$uid" > test.txt # adding new user (pyros_user) and creating his home folder -RUN useradd --create-home --shell /bin/bash pyros_user +RUN useradd --create-home --shell /bin/bash pyros_user --uid $uid # Create the work dir and set permissions as pyros_user RUN mkdir -p /home/pyros_user/app/ && chown -R pyros_user:pyros_user /home/pyros_user/app diff --git a/VERSION b/VERSION index 474220c..5819300 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.6.0 \ No newline at end of file +0.3.7.0 \ No newline at end of file diff --git a/docker/PYROS_DOCKER_BUILD.bat b/docker/PYROS_DOCKER_BUILD.bat index 846d6d7..c531dbe 100755 --- a/docker/PYROS_DOCKER_BUILD.bat +++ b/docker/PYROS_DOCKER_BUILD.bat @@ -1,2 +1,2 @@ -docker-compose build +CURRENT_UID=$(id -u) docker-compose build diff --git a/docker/PYROS_DOCKER_INSTALL b/docker/PYROS_DOCKER_INSTALL index 6e73cba..9ac7ae3 100755 --- a/docker/PYROS_DOCKER_INSTALL +++ b/docker/PYROS_DOCKER_INSTALL @@ -15,6 +15,8 @@ fi while ! docker inspect db_pyros --format='{{.State.Health.Status}}' | grep -q 'healthy' do sleep 5 + heal_status=$(docker inspect db_pyros --format='{{.State.Health.Status}}') + echo "Current status : $heal_status" done # db container is ready to execture queries, we can start the installation diff --git a/docker/PYROS_DOCKER_START.bat b/docker/PYROS_DOCKER_START.bat index c214678..f1a3593 100755 --- a/docker/PYROS_DOCKER_START.bat +++ b/docker/PYROS_DOCKER_START.bat @@ -1,4 +1,4 @@ docker-compose down --remove-orphans -docker-compose up -d +CURRENT_UID=$(id -u) docker-compose up -d diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 55bfabb..4eea69e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -3,7 +3,8 @@ version: "3.9" services: db: # if we're using mysql >= 8, some of sql queries aren't valid anymore, like for creating and grant an user at the same time - image: mysql:5.7.22 + # This is fixed in pyros.py + image: mysql:8.0.28 command: --default-authentication-plugin=mysql_native_password restart: always container_name: db_pyros @@ -24,12 +25,6 @@ services: # service image of python, that let users to interact with python scripts such as pyros. pyros: # path to the Dockerfile of this image - build: .. - container_name: pyros - # tty is the -t option in docker exec - tty: true - # stdin_open is the -i option in docker exec - stdin_open: true env_file: - variables.env environment: @@ -42,6 +37,15 @@ services: - "MYSQL_PYROS_LOGIN=${MYSQL_PYROS_LOGIN:-pyros}" - "MYSQL_PYROS_PWD=${MYSQL_PYROS_PWD:-DjangoPyros}" #- "IP_PYROS_USER=${IP_PYROS_USER:-172.28.1.5}" + build: + args: + uid: ${CURRENT_UID} + context: .. + container_name: pyros + # tty is the -t option in docker exec + tty: true + # stdin_open is the -i option in docker exec + stdin_open: true volumes: - ..:/home/pyros_user/app # tells which port of local machine can communicate with the docker image (host:container), host is your local machine diff --git a/pyros.py b/pyros.py index e788b00..e96c447 100755 --- a/pyros.py +++ b/pyros.py @@ -21,8 +21,6 @@ import sys import time import re import glob -from config.pyros.config_pyros import configpyros -from git import Repo #import logging @@ -345,9 +343,9 @@ def set_environment_variables_if_not_configured(env_path:str, env_sample_path:st if WITH_DOCKER and os.environ.get(key) != value: log.warning(f"WARNING: Environment value for '{key}' needs to be uptaded (To remove this message : restart docker container to update them).'") else: - if not WITH_DOCKER and os.environ.get(key) is None: - log.error(f"ERROR: Error while reading value for '{key}'.Please fix the '{env_file}' file. For this run, PyROS will take the values from '{env_path}'") - raise BaseException() + # if not WITH_DOCKER and os.environ.get(key) is None: + # log.error(f"ERROR: Error while reading value for '{key}'.Please fix the '{env_file}' file. For this run, PyROS will take the values from '{env_path}'") + # raise BaseException() if WITH_DOCKER and os.environ.get(key) is None: log.warning(f"WARNING: Environment value for '{key}' isn't defined within the container (To remove this message : restart docker container to update environment values). PyROS will take the values from '{env_path}'") raise BaseException() @@ -519,6 +517,18 @@ def install_or_update(UPDATE:bool=False, packages_only:bool=False, database_only print("- packages_only:", packages_only) print("- database_only:", database_only) # Git clone Guitastro if not already cloned: + try: + from git import Repo + except: + pip = "pip" if IS_WINDOWS else "pip3" + process = subprocess.Popen(pip + " install --upgrade GitPython", shell=True) + process.wait() + if process.returncode == 0: + # self.addExecuted(self.current_command, command) + from git import Repo + else: + log.error("GitPython package (required for obsconfig class) installation failed") + if not os.path.exists( os.path.join(os.getcwd(),"./vendor/guitastro")): print("Cloning Guiastro repository") cloned_repo = Repo.clone_from("https://gitlab.irap.omp.eu/aklotz/guitastro.git", os.path.join(os.getcwd(),"./vendor/guitastro")) @@ -677,6 +687,19 @@ def initdb(): #def start(agent:str, configfile:str, test, verbosity): def start(agent:str, configfile:str, observatory:str, unit:str): log.debug("Running start command") + try: + from config.pyros.config_pyros import ConfigPyros + except: + #pip = "pip" if platform.system() == "Windows" else "pip3" + # TODO: "python -m pip" au lieu de "pip" + pip = "pip" if IS_WINDOWS else "pip3" + process = subprocess.Popen(pip + " install --upgrade pykwalify", shell=True) + process.wait() + if process.returncode == 0: + # self.addExecuted(self.current_command, command) + from config.pyros.config_pyros import configpyros + else: + log.error("pykwalify package (required for obsconfig class) installation failed") if configfile: log.debug("With config file"+ configfile) os.environ["pyros_config_file"] = os.path.join(os.path.abspath(PYROS_DJANGO_BASE_DIR),"../../../config/pyros/", configfile) @@ -684,7 +707,7 @@ def start(agent:str, configfile:str, observatory:str, unit:str): configfile = 'config_pyros.yml' os.environ["pyros_config_file"] = os.path.join(os.path.abspath(PYROS_DJANGO_BASE_DIR),"../../../config/pyros/", configfile) - logo_name = configpyros(os.environ["pyros_config_file"]).pyros_config["general"]["logo"] + logo_name = ConfigPyros(os.environ["pyros_config_file"]).pyros_config["general"]["logo"] logo_path = os.path.join(os.path.abspath(PYROS_DJANGO_BASE_DIR),"../../../config/pyros/", logo_name) media_path = os.path.join(os.path.abspath(PYROS_DJANGO_BASE_DIR),"misc/static/media",logo_name) shutil.copy(logo_path,media_path) @@ -1103,9 +1126,10 @@ def install_packages(): # Pip upgrade wheel and setuptools venv_pip_install('wheel', '--upgrade') #os.system(VENV_PIP+' install --upgrade wheel') - venv_pip_install('setuptools', '--upgrade') - #os.system(VENV_PIP+' install --upgrade setuptools') - + # Not working with python 3.8 (on 17/02/2022) + # venv_pip_install('setuptools', '--upgrade') + os.system(VENV_PIP+' install setuptools==58') + # Pip install required packages from REQUIREMENTS file print() print(Colors.LOG_BLUE + "-----------------------------Installing python packages via pip-----------------------------" + Colors.END) @@ -1156,7 +1180,7 @@ def install_database(venv): # -------------------------------------------- output = subprocess.check_output("mysql --version", shell=True) # output is something like: "mysql Ver 15.1 Distrib 10.0.20-MariaDB, for Linux (x86_64) using EditLine wrapper" - tmp = (str(output).split()[4]).split('.') + tmp = (str(output).split()[2]).split('.') sql_version = float(tmp[0]+'.'+tmp[1]) print(Colors.LOG_BLUE + "MySQL version is " + str(sql_version) + Colors.END) @@ -1184,7 +1208,8 @@ def install_database(venv): IF_EXISTS = '' if sql_version < 5.5 else 'IF EXISTS' sql_query = \ f"DROP DATABASE {IF_EXISTS} {SQL_DATABASE}; CREATE DATABASE {SQL_DATABASE}; " +\ - f"DROP DATABASE {IF_EXISTS} {SQL_DATABASE_SIMU}; CREATE DATABASE {SQL_DATABASE_SIMU}; " + f"DROP DATABASE {IF_EXISTS} {SQL_DATABASE_SIMU}; CREATE DATABASE {SQL_DATABASE_SIMU}; " +\ + f"DROP DATABASE {IF_EXISTS} {SQL_DATABASE_TEST}; CREATE DATABASE {SQL_DATABASE_TEST}; " # 2) Create user pyros and give it all rights on 3 databases # Ne marche pas si l'utilisateur existe déjà => erreur @@ -1201,11 +1226,22 @@ def install_database(venv): host = "%" else: host = "localhost" - - sql_query += f"GRANT ALL ON {SQL_DATABASE}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; " - sql_query += f"GRANT ALL ON {SQL_DATABASE_SIMU}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; " - # This database does not yet exists and will be automatically created by Django, but we already give access rights to it for pyros user - sql_query += f"GRANT ALL ON {SQL_DATABASE_TEST}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; " + # FOR MYSQL 8: + if sql_version >=8: + # We remove identified by because grant was creating the pyros user, wasn't working on version 8 so we separated the steps : + # first we create the user then we grant him rights + sql_query += f"CREATE USER IF NOT EXISTS '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; " + sql_query += f"GRANT ALL ON {SQL_DATABASE}.* TO '{SQL_USER}'@'{host}'; " + sql_query += f"GRANT ALL ON {SQL_DATABASE_SIMU}.* TO '{SQL_USER}'@'{host}'; " + sql_query += f"GRANT ALL ON {SQL_DATABASE_TEST}.* TO '{SQL_USER}'@'{host}'; " + else: + sql_query += f"GRANT ALL ON {SQL_DATABASE}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; " + sql_query += f"GRANT ALL ON {SQL_DATABASE_SIMU}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; " + # This database does not yet exists and will be automatically created by Django, but we already give access rights to it for pyros user + sql_query += f"GRANT ALL ON {SQL_DATABASE_TEST}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; " + # FOR MYSQL 8: + if sql_version >= 8: + sql_query += "FLUSH PRIVILEGES; " #"GRANT USAGE ON *.* TO '"+SQL_USER+"'@localhost ; " ''' @@ -1247,7 +1283,7 @@ def install_database(venv): mysql_host = "localhost" # when using -p option, there is no space between the option and the password value - mysql_call_root = f'mysql -h {mysql_host} -u {mysql_login} -p{mysql_root_password}' + mysql_call_root = f"mysql -h {mysql_host} -u '{mysql_login}' -p'{mysql_root_password}'" mysql_call_pyros = "\"" + MYSQL_EXE_PATH+ "mysql\" -u "+SQL_USER+" -p" # -------------------------------------------- -- libgit2 0.21.2