Upgrading mysql version and fixing sql queries

Alexis Koralewski
1 parent 46b415a3
Showing 8 changed files with 82 additions and 29 deletions Show diff stats
CHANGELOG
Dockerfile
VERSION
docker/PYROS_DOCKER_BUILD.bat
docker/PYROS_DOCKER_INSTALL
docker/PYROS_DOCKER_START.bat
docker/docker-compose.yml
pyros.py
+18-02-2022 (AKo): v0.3.7.0
+	- Upgrade Mysql version 
+	- Fix sql queries for mysql 8.0
+	- Add pyros_user in sequence list and detail
+	- Add description field in observatories configuration 
+	- Add those description as tooltips when they are displayed sequence fields 
+	- Add local fontawesome css
+	- Rename configpyros properly (ConfigPyros)
+	- Fix dockerfile issue with permission (uid needs to be the same as the host)
+
 08-02-2022 (AKo) : v0.3.6.0
 	- Add DRF (Django Rest Framework) to requirements
 	- Add Pyros API (User and sequences with sequence submission)
 FROM python:3.8
+ARG uid
 RUN apt-get update && apt-get install -y \
     default-mysql-client \
@@ -9,9 +10,9 @@ RUN apt-get update &amp;&amp; apt-get install -y \
 # Get IRAP self signed certificate
 RUN echo | openssl s_client -connect gitlab.irap.omp.eu:443 -servername gitlab.irap.omp.eu 2>/dev/null | openssl x509 > /etc/ssl/certs/gitlab.irap.omp.eu.crt
-
+RUN echo "$uid" > test.txt
 # adding new user (pyros_user) and creating his home folder
-RUN useradd --create-home --shell /bin/bash pyros_user
+RUN useradd --create-home --shell /bin/bash pyros_user --uid $uid
 # Create the work dir and set permissions as pyros_user 
 RUN mkdir -p /home/pyros_user/app/ && chown -R pyros_user:pyros_user /home/pyros_user/app
-0.3.6.0
 \ No newline at end of file
+0.3.7.0
 \ No newline at end of file
-docker-compose build
+CURRENT_UID=$(id -u) docker-compose build
@@ -15,6 +15,8 @@ fi
 while ! docker inspect db_pyros --format='{{.State.Health.Status}}' | grep -q 'healthy'
 do
     sleep 5
+    heal_status=$(docker inspect db_pyros --format='{{.State.Health.Status}}')
+    echo "Current status : $heal_status"
 done
 # db container is ready to execture queries, we can start the installation
 docker-compose down --remove-orphans
-docker-compose up -d
+CURRENT_UID=$(id -u) docker-compose up -d
@@ -3,7 +3,8 @@ version: &quot;3.9&quot;
 services:
   db:
     # if we're using mysql >= 8, some of sql queries aren't valid anymore, like for creating and grant an user at the same time
-    image: mysql:5.7.22
+    # This is fixed in pyros.py
+    image: mysql:8.0.28
     command: --default-authentication-plugin=mysql_native_password
     restart: always
     container_name: db_pyros
@@ -24,12 +25,6 @@ services:
   # service image of python, that let users to interact with python scripts such as pyros.
   pyros:
     # path to the Dockerfile of this image
-    build: ..
-    container_name: pyros
-    # tty is the -t option in docker exec 
-    tty: true
-    # stdin_open is the -i option in docker exec
-    stdin_open: true
     env_file: 
       - variables.env
     environment:
@@ -42,6 +37,15 @@ services:
       - "MYSQL_PYROS_LOGIN=${MYSQL_PYROS_LOGIN:-pyros}"
       - "MYSQL_PYROS_PWD=${MYSQL_PYROS_PWD:-DjangoPyros}"
       #- "IP_PYROS_USER=${IP_PYROS_USER:-172.28.1.5}"
+    build: 
+      args:
+        uid: ${CURRENT_UID}
+      context: ..
+    container_name: pyros
+    # tty is the -t option in docker exec 
+    tty: true
+    # stdin_open is the -i option in docker exec
+    stdin_open: true
     volumes:
       - ..:/home/pyros_user/app
     # tells which port of local machine can communicate with the docker image (host:container), host is your local machine
@@ -21,8 +21,6 @@ import sys
 import time
 import re
 import glob
-from config.pyros.config_pyros import configpyros
-from git import Repo
 #import logging
@@ -345,9 +343,9 @@ def set_environment_variables_if_not_configured(env_path:str, env_sample_path:st
                     if WITH_DOCKER and os.environ.get(key) != value:
                         log.warning(f"WARNING: Environment value for '{key}' needs to be uptaded (To remove this message : restart docker container to update them).'")
                     else:
-                        if not WITH_DOCKER and os.environ.get(key) is None:
-                            log.error(f"ERROR: Error while reading value for '{key}'.Please fix the '{env_file}' file. For this run, PyROS will take the values from '{env_path}'")    
-                            raise BaseException()
+                        # if not WITH_DOCKER and os.environ.get(key) is None:
+                        #     log.error(f"ERROR: Error while reading value for '{key}'.Please fix the '{env_file}' file. For this run, PyROS will take the values from '{env_path}'")    
+                        #     raise BaseException()
                         if WITH_DOCKER and os.environ.get(key) is None:
                             log.warning(f"WARNING: Environment value for '{key}' isn't defined within the container (To remove this message : restart docker container to update environment values). PyROS will take the values from '{env_path}'")    
                             raise BaseException()
@@ -519,6 +517,18 @@ def install_or_update(UPDATE:bool=False, packages_only:bool=False, database_only
         print("- packages_only:", packages_only)
         print("- database_only:", database_only)
         # Git clone Guitastro if not already cloned:
+        try:
+            from git import Repo
+        except:
+            pip = "pip" if IS_WINDOWS else "pip3"
+            process = subprocess.Popen(pip + " install --upgrade GitPython", shell=True)
+            process.wait()
+            if process.returncode == 0:
+            # self.addExecuted(self.current_command, command)
+                from git import Repo
+            else:
+                log.error("GitPython package (required for obsconfig class) installation failed")
+
         if not os.path.exists( os.path.join(os.getcwd(),"./vendor/guitastro")):
             print("Cloning Guiastro repository")
             cloned_repo = Repo.clone_from("https://gitlab.irap.omp.eu/aklotz/guitastro.git", os.path.join(os.getcwd(),"./vendor/guitastro"))
@@ -677,6 +687,19 @@ def initdb():
 #def start(agent:str, configfile:str, test, verbosity):
 def start(agent:str, configfile:str, observatory:str, unit:str):
     log.debug("Running start command")
+    try:
+        from config.pyros.config_pyros import ConfigPyros
+    except:
+        #pip = "pip" if platform.system() == "Windows" else "pip3"
+        # TODO: "python -m pip" au lieu de "pip"
+        pip = "pip" if IS_WINDOWS else "pip3"
+        process = subprocess.Popen(pip + " install --upgrade pykwalify", shell=True)
+        process.wait()
+        if process.returncode == 0:
+            # self.addExecuted(self.current_command, command)
+            from config.pyros.config_pyros import configpyros
+        else:
+            log.error("pykwalify package (required for obsconfig class) installation failed")
     if configfile: 
         log.debug("With config file"+ configfile)
         os.environ["pyros_config_file"] = os.path.join(os.path.abspath(PYROS_DJANGO_BASE_DIR),"../../../config/pyros/", configfile)
@@ -684,7 +707,7 @@ def start(agent:str, configfile:str, observatory:str, unit:str):
         configfile = 'config_pyros.yml'
         os.environ["pyros_config_file"] = os.path.join(os.path.abspath(PYROS_DJANGO_BASE_DIR),"../../../config/pyros/", configfile)
-    logo_name = configpyros(os.environ["pyros_config_file"]).pyros_config["general"]["logo"]
+    logo_name = ConfigPyros(os.environ["pyros_config_file"]).pyros_config["general"]["logo"]
     logo_path = os.path.join(os.path.abspath(PYROS_DJANGO_BASE_DIR),"../../../config/pyros/", logo_name)
     media_path = os.path.join(os.path.abspath(PYROS_DJANGO_BASE_DIR),"misc/static/media",logo_name)
     shutil.copy(logo_path,media_path)
@@ -1103,9 +1126,10 @@ def install_packages():
     # Pip upgrade wheel and setuptools
     venv_pip_install('wheel', '--upgrade')
     #os.system(VENV_PIP+' install --upgrade wheel')
-    venv_pip_install('setuptools', '--upgrade')
-    #os.system(VENV_PIP+' install --upgrade setuptools')
-    
+    # Not working with python 3.8 (on 17/02/2022)
+    # venv_pip_install('setuptools', '--upgrade')
+    os.system(VENV_PIP+' install setuptools==58')
+         
     # Pip install required packages from REQUIREMENTS file
     print()
     print(Colors.LOG_BLUE + "-----------------------------Installing python packages via pip-----------------------------" + Colors.END)
@@ -1156,7 +1180,7 @@ def install_database(venv):
     # --------------------------------------------
     output = subprocess.check_output("mysql --version", shell=True)
     # output is something like: "mysql  Ver 15.1 Distrib 10.0.20-MariaDB, for Linux (x86_64) using  EditLine wrapper"
-    tmp = (str(output).split()[4]).split('.')
+    tmp = (str(output).split()[2]).split('.')
     sql_version = float(tmp[0]+'.'+tmp[1])
     print(Colors.LOG_BLUE + "MySQL version is " + str(sql_version) + Colors.END)
@@ -1184,7 +1208,8 @@ def install_database(venv):
     IF_EXISTS = '' if sql_version < 5.5 else 'IF EXISTS'
     sql_query = \
         f"DROP DATABASE {IF_EXISTS} {SQL_DATABASE}; CREATE DATABASE {SQL_DATABASE}; " +\
-        f"DROP DATABASE {IF_EXISTS} {SQL_DATABASE_SIMU}; CREATE DATABASE {SQL_DATABASE_SIMU}; "
+        f"DROP DATABASE {IF_EXISTS} {SQL_DATABASE_SIMU}; CREATE DATABASE {SQL_DATABASE_SIMU}; " +\
+        f"DROP DATABASE {IF_EXISTS} {SQL_DATABASE_TEST}; CREATE DATABASE {SQL_DATABASE_TEST}; "
     # 2) Create user pyros and give it all rights on 3 databases 
     # Ne marche pas si l'utilisateur existe déjà => erreur
@@ -1201,11 +1226,22 @@ def install_database(venv):
         host = "%"
     else:
         host = "localhost"
-
-    sql_query += f"GRANT ALL ON {SQL_DATABASE}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; "
-    sql_query += f"GRANT ALL ON {SQL_DATABASE_SIMU}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; "
-    # This database does not yet exists and will be automatically created by Django, but we already give access rights to it for pyros user 
-    sql_query += f"GRANT ALL ON {SQL_DATABASE_TEST}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; "
+    # FOR MYSQL 8:
+    if sql_version >=8:
+        # We remove identified by because grant was creating the pyros user, wasn't working on version 8 so we separated the steps :
+        # first we create the user then we grant him rights
+        sql_query += f"CREATE USER IF NOT EXISTS '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; "
+        sql_query += f"GRANT ALL ON {SQL_DATABASE}.* TO '{SQL_USER}'@'{host}'; "
+        sql_query += f"GRANT ALL ON {SQL_DATABASE_SIMU}.* TO '{SQL_USER}'@'{host}'; "
+        sql_query += f"GRANT ALL ON {SQL_DATABASE_TEST}.* TO '{SQL_USER}'@'{host}'; "
+    else:
+        sql_query += f"GRANT ALL ON {SQL_DATABASE}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; "
+        sql_query += f"GRANT ALL ON {SQL_DATABASE_SIMU}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; "
+        # This database does not yet exists and will be automatically created by Django, but we already give access rights to it for pyros user 
+        sql_query += f"GRANT ALL ON {SQL_DATABASE_TEST}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; "
+    # FOR MYSQL 8:
+    if sql_version >= 8:
+        sql_query += "FLUSH PRIVILEGES; "
     #"GRANT USAGE ON *.* TO '"+SQL_USER+"'@localhost ; "
     '''
@@ -1247,7 +1283,7 @@ def install_database(venv):
         mysql_host = "localhost"
     # when using -p option, there is no space between the option and the password value
-    mysql_call_root = f'mysql -h {mysql_host} -u {mysql_login} -p{mysql_root_password}'
+    mysql_call_root = f"mysql -h {mysql_host} -u '{mysql_login}' -p'{mysql_root_password}'"
     mysql_call_pyros = "\"" + MYSQL_EXE_PATH+ "mysql\" -u "+SQL_USER+" -p"
     # --------------------------------------------