Merge branch 'dev' of https://gitlab.irap.omp.eu/pyros-irap/pyros into dev

Alexis Koralewski
2 parents b878d67a 8ca1d24c
Showing 6 changed files with 60 additions and 52 deletions Show diff stats
docker/Dockerfile
docker/PYROS_DOCKER_BUILD
docker/PYROS_DOCKER_INSTALL_DB
docker/PYROS_DOCKER_UPDATE
docker/docker-compose.yml
pyros.py
@@ -171,6 +171,8 @@ RUN useradd -m -s /bin/bash pyros_user --uid $uid
  
 # Create the work dir and set permissions as pyros_user 
 RUN mkdir -p /home/pyros_user/app/ && chown -R pyros_user:pyros_user /home/pyros_user/app
+
+# cd /home/pyros_user/app
 WORKDIR /home/pyros_user/app
  
 # Switch from root to pyros_user
@@ -184,6 +186,8 @@ RUN mkdir -p ./vendor/guitastro/install/
 RUN mkdir -p ./install/
  
 # Copy local host machine files to image
+# Dockerfile contains this volume => ../..:/home/pyros_user/app
+# Thus, this will copy PYROS_SOFT/* ==TO==> /home/pyros_user/app/ (the WORKDIR)
 COPY ./vendor/guitastro/install/requirements.in ./vendor/guitastro/install/requirements.in
 COPY ./install/requirements.txt ./install/requirements.txt 
 COPY ./install/requirements_dev.txt ./install/requirements_dev.txt 
@@ -193,9 +197,9 @@ ENV PATH &quot;$PATH:/home/pyros_user/.local/bin&quot;
  
  
 # Installing click on the image to prevent error on the first execution of the installation script
-RUN python3 -m pip  install --user click
+RUN python3 -m pip install --user click
  
-RUN python3 -m pip  install --user setuptools==58
+RUN python3 -m pip install --user setuptools==58
  
 # (EP 23/3/2022) Installing pip-tools for the management of all the requirements*.txt files (python dependencies packages)
 # NB :
@@ -204,14 +208,14 @@ RUN python3 -m pip  install --user setuptools==58
 # - Unfortunatly, it is difficult to use with several requirements*.txt files as it is the case for this software : pyros + sphinx + guitastro...
 # - So we cannot yet use it completely and still have to use the traditional "pip install -r" anyway ...
 # - But we can at least use it to generate all the requirements*.txt files in a far better format
-RUN python3 -m pip  install --user pip-tools
+RUN python3 -m pip install --user pip-tools
  
 # Installing packages required for Guitastro
 RUN pip-compile ./vendor/guitastro/install/requirements.in 
-RUN python3 -m pip  install --user -r ./vendor/guitastro/install/requirements.txt
+RUN python3 -m pip install --user -r ./vendor/guitastro/install/requirements.txt
 # Maybe unnecessary because same requirements as for pyros requirements_dev (?)
 #RUN pip install --user -r ./vendor/guitastro/install/requirements_dev.txt
  
 # Installing packages required for PyROS
-RUN python3 -m pip  install --user -r ./install/requirements.txt
-RUN python3 -m pip  install --user -r ./install/requirements_dev.txt
+RUN python3 -m pip install --user -r ./install/requirements.txt
+RUN python3 -m pip install --user -r ./install/requirements_dev.txt
 #!/usr/bin/env bash
  
+# Git clone guitastro
+#
+# This could be done directly in the Dockerfile
+# but it is better to do it here because if it failed during the build 
+# we would have to do all the build again...
 [ ! -d "../vendor/guitastro/" ] && git clone https://gitlab.irap.omp.eu/guitastrolib/guitastro.git ../vendor/guitastro/
+[ ! -d "../vendor/guitastro/" ] && echo "Error : git clone of GuitAstro failed => cannot go on with the build" && exit 1
  
 export CURRENT_UID=$(id -u)
 export COMPUTER_HOSTNAME=$HOSTNAME
 #!/usr/bin/env bash
  
-export CURRENT_UID=$(id -u)
-export COMPUTER_HOSTNAME=$HOSTNAME
+# PRE-CONDITION : pyros container must be running
  
 # if no container is running
 if ! [ $(docker ps | grep 'pyros' | wc -l) -eq 4 ] ; then
 #!/usr/bin/env bash
  
-export CURRENT_UID=$(id -u)
-export COMPUTER_HOSTNAME=$HOSTNAME
+# PRE-CONDITION : pyros container must be running
  
 # If no container is running Start it
 if ! [ $(docker ps | grep 'pyros' | wc -l) -eq 4 ]
@@ -19,10 +18,12 @@ echo &quot;Updating Guitastro source code&quot;
 echo "Updating PyROS source code"
 git pull
  
-echo "Updating observatory source code"
-for dir in ../../PYROS_OBSERVATORY/*
-do
-    git pull
+# 3) Update all observatories with git repo
+for dir in ../../PYROS_OBSERVATORY/* ; do
+    if [ -d .git ] ; then 
+        echo "Updating observatory $dir source code"
+        git pull
+    fi
 done
  
 # 3) pyros.py update => update BD + doc + Guitastro requirements
@@ -111,6 +111,7 @@ services:
     tty: true
     # stdin_open is the -i option in docker exec
     stdin_open: true
+    # Local PYROS_SOFT/ folder is linked to container /home/pyros_user/app folder
     volumes:
       - ../..:/home/pyros_user/app
     # tells which port of local machine can communicate with the docker image (host:container), host is your local machine
@@ -534,7 +534,7 @@ def dbshell():
  
 @pyros_launcher.command(help="Update (only if necessary) the python packages AND the source code AND the DB structure")
 def update():
-    install_or_update(UPDATE=True)
+    _install_or_update(UPDATE=True)
     '''
     print("Running update command")
     # 1) Update source code (git pull)
@@ -568,13 +568,14 @@ def update():
 def install(packages_only, database_only):
     with_packages = not database_only
     with_database = not packages_only
-    install_or_update(UPDATE=False, with_packages=with_packages,
+    _install_or_update(UPDATE=False, with_packages=with_packages,
                       with_database=with_database)
     # install_or_update(UPDATE=False, with_packages=packages_only, with_database=database_only)
     # install_or_update(UPDATE=False, packages_only=packages_only, database_only=database_only)
  
+
 # def install_or_update(UPDATE: bool = False, with_packages: bool = False, with_database: bool = False):
-def install_or_update(UPDATE: bool = False, with_packages: bool = True, with_database: bool = True):
+def _install_or_update(UPDATE: bool = False, with_packages: bool = True, with_database: bool = True):
     SQL_USER = os.environ.get("MYSQL_PYROS_LOGIN").strip()
     SQL_PSWD = os.environ.get("MYSQL_PYROS_PWD").strip()
     os.environ["PATH_TO_OBSCONF_FOLDER"] = os.path.join(os.path.abspath(
@@ -583,7 +584,7 @@ def install_or_update(UPDATE: bool = False, with_packages: bool = True, with_dat
             PYROS_DJANGO_BASE_DIR), os.environ["PATH_TO_OBSCONF_FOLDER"] + "observatory.yml")
     ACTION = "UPDATING" if UPDATE else "INSTALLING"
  
-    # if WITH_DOCKER: with_database = True
+    # if WITH_DOCKER: only install the DB (with_database = True)
     if WITH_DOCKER:
         with_packages = False
  
@@ -597,21 +598,22 @@ def install_or_update(UPDATE: bool = False, with_packages: bool = True, with_dat
     # if (os.path.basename(os.getcwd()) != "private"):
  
     num = 0
-    # 1) Update source code (git pull)
+
     if UPDATE:
+        # Update source code (git pull)
         num += 1
-        printFullTerm(
-            Colors.BLUE, f"{num}) UPDATING SOURCE CODE: Running git pull")
+        printFullTerm(Colors.BLUE, f"{num}) UPDATING SOURCE CODE: Running git pull")
         if not WITH_DOCKER:
             _gitpull() or die()
  
-    # 2) Update python packages (pip upgrade AND pip install requirements)
     if with_packages:
+        # Update python packages (pip upgrade AND pip install requirements)
         num += 1
         printFullTerm(Colors.BLUE, f"{num}) {ACTION} PYTHON PACKAGES")
         # (UPDATE) Re-install VENV if disappeared
         install_venv(EVEN_IF_ALREADY_EXISTS=not UPDATE)
         install_packages()
+
     if UPDATE:
         print("Running UPDATE command")
     else:
@@ -621,13 +623,10 @@ def install_or_update(UPDATE: bool = False, with_packages: bool = True, with_dat
             from git import Repo
         except:
             pip = "pip" if IS_WINDOWS else "pip3"
-            if WITH_DOCKER:
-                process = subprocess.Popen(
-                    pip + " install --user --upgrade GitPython", shell=True)
-            else:
-                process = subprocess.Popen(
-                    pip + " install --upgrade GitPython", shell=True)
+            usermode = '--user' if WITH_DOCKER else ''
+            process = subprocess.Popen(pip + f" install {usermode} --upgrade GitPython", shell=True)
             process.wait()
+            # if run ok
             if process.returncode == 0:
                 # self.addExecuted(self.current_command, command)
                 from git import Repo
@@ -645,14 +644,16 @@ def install_or_update(UPDATE: bool = False, with_packages: bool = True, with_dat
         change_dir("PREVIOUS")
         '''
  
+    #
     # Guitastro
+    #
     GUITASTRO_PATH = os.path.join(os.getcwd(), "./vendor/guitastro")
-    if with_packages and not WITH_DOCKER :
+    if with_packages:  # and not WITH_DOCKER:
         #GUITASTRO_PATH = os.path.join(os.getcwd(), "../vendor/guitastro")
         change_dir("..")
     print(os.getcwd())
     # 1) clone repo if not yet done
-    if not os.path.exists(GUITASTRO_PATH) and not WITH_DOCKER :
+    if not WITH_DOCKER and not os.path.exists(GUITASTRO_PATH):
         print("Guitastro : Cloning repository")
         cloned_repo = Repo.clone_from(
             "https://gitlab.irap.omp.eu/guitastrolib/guitastro.git", GUITASTRO_PATH)
@@ -661,7 +662,7 @@ def install_or_update(UPDATE: bool = False, with_packages: bool = True, with_dat
         gitpull_guitastro()
         change_dir("PREVIOUS")
     # 2) install/update requirements & generate API doc
-    if os.path.exists(GUITASTRO_PATH) and not WITH_DOCKER:
+    if not WITH_DOCKER and os.path.exists(GUITASTRO_PATH):
         # TODO: update guitastro (git pull from vendor/guitastro/)
         print("\nGuitastro : Installing/Updating python package dependencies\n")
         # Upgrade pip if new version available
@@ -703,9 +704,8 @@ def install_or_update(UPDATE: bool = False, with_packages: bool = True, with_dat
             _updatedb() or die()
         else:
             res = install_database(VENV)
-            if(res == 0):
-                print(
-                    f"You can connect to PyROS as '{SQL_USER}' user with the password '{SQL_PSWD}' ")
+            if (res == 0):
+                print(f"You can connect to PyROS as '{SQL_USER}' user with the password '{SQL_PSWD}' ")
     return True
  
  
@@ -789,8 +789,7 @@ def gitpull_guitastro():
     change_dir("./vendor/guitastro/")
     GIT = "git.exe" if IS_WINDOWS else "git"
     if not test_mode():
-        return execProcess(f"{GIT} pull",foreground=True)
-    
+        return execProcess(f"{GIT} pull", foreground=True)
     return True
  
  
@@ -816,21 +815,24 @@ def initdb():
     return True
  
  
-
-@pyros_launcher.command(help="Install dependencies")
+@pyros_launcher.command(help="Install dependencies from Observatory config DEPENDENCY section")
 # @global_test_options
-@click.option('--observatory', '-o', help='the observatory name to be used')
-@click.option('--unit', '-u', help='the unit name to be used')
+@click.option('--observatory', '-o', help='the Observatory name to be used')
+@click.option('--unit', '-u', help='the Unit name to be used')
 @click.option("--foreground","-fg", is_flag=True, help="Print stdout and error in terminal")
 def install_dependencies(observatory: str, unit: str, foreground: bool):
-    if observatory == None or len(observatory) == 0:
+    if (observatory is None) or (len(observatory) == 0):
         observatory = "default"
         default_obsconfig_folder = os.path.join(
-            os.path.abspath(PYROS_DJANGO_BASE_DIR), "../../../config/pyros_observatory/")
+            os.path.abspath(PYROS_DJANGO_BASE_DIR),
+            "../../../config/pyros_observatory/"
+        )
         path_to_obs_config_folder = default_obsconfig_folder+"pyros_observatory_"+observatory+"/"
     else:
         observatories_configuration_folder = os.path.join(
-            os.path.abspath(PYROS_DJANGO_BASE_DIR), "../../../../PYROS_OBSERVATORY/")
+            os.path.abspath(PYROS_DJANGO_BASE_DIR),
+            "../../../../PYROS_OBSERVATORY/"
+        )
         if len(glob.glob(observatories_configuration_folder+"pyros_observatory_"+observatory+"/")) != 1:
             # Observatory configuration folder not found
             print(
@@ -845,15 +847,14 @@ def install_dependencies(observatory: str, unit: str, foreground: bool):
     # Search for observatory config file
     obs_config_file_name = glob.glob(os.path.join(path_to_obs_config_folder, "observatory.yml"))[0]
  
-    obs_config_file_path = os.path.join(
-        path_to_obs_config_folder, obs_config_file_name)
+    obs_config_file_path = os.path.join(path_to_obs_config_folder, obs_config_file_name)
     os.environ["PATH_TO_OBSCONF_FILE"] = obs_config_file_path
     os.environ["PATH_TO_OBSCONF_FOLDER"] = path_to_obs_config_folder
     os.environ["unit_name"] = unit if unit else ''
  
     # add path to pyros_django folder as the config class is supposed to work within this folder
     cmd_test_obs_config = f"-c \"from src.core.pyros_django.obs_config.obsconfig_class import OBSConfig\nOBSConfig('{obs_config_file_path}')\""
-    if not execProcessFromVenv(cmd_test_obs_config,foreground=True):
+    if not execProcessFromVenv(cmd_test_obs_config, foreground=True):
         # Observatory configuration has an issue
         exit(1)
     else:
@@ -862,12 +863,8 @@ def install_dependencies(observatory: str, unit: str, foreground: bool):
             from git import Repo, GitError
         except:
             pip = "pip" if IS_WINDOWS else "pip3"
-            if WITH_DOCKER:
-                process = subprocess.Popen(
-                    pip + " install --user --upgrade GitPython", shell=True)
-            else:
-                process = subprocess.Popen(
-                    pip + " install --upgrade GitPython", shell=True)
+            usermode = '--user' if WITH_DOCKER else ''
+            process = subprocess.Popen(pip + f" install {usermode} --upgrade GitPython", shell=True)
             process.wait()
             if process.returncode == 0:
                 # self.addExecuted(self.current_command, command)