PYROS-IRAP / pyros

Commit 4c0f2742bdca25329733ebab91a494432308a8a5

Authored by Alain Klotz
2 parents 0233a388 8a7ec6c9
Exists in dev

Merge branch 'dev' of https://gitlab.irap.omp.eu/pyros-irap/pyros into dev

Showing 23 changed files with 410 additions and 302 deletions   Show diff stats
Inline Side-by-side
data/README.md deleted
View file @0233a38
... ... @@ -1 +0,0 @@
1   -Folder to place Agents data
2 0 \ No newline at end of file
docker/Dockerfile
  Diff comments   View file @4c0f274
... ... @@ -171,6 +171,8 @@ RUN useradd -m -s /bin/bash pyros_user --uid $uid
171 171  
172 172 # Create the work dir and set permissions as pyros_user
173 173 RUN mkdir -p /home/pyros_user/app/ && chown -R pyros_user:pyros_user /home/pyros_user/app
  174 +
  175 +# cd /home/pyros_user/app
174 176 WORKDIR /home/pyros_user/app
175 177  
176 178 # Switch from root to pyros_user
... ... @@ -180,21 +182,24 @@ USER pyros_user
180 182 # TODO: pourquoi numpy ??? A virer ?
181 183 #RUN pip install --user numpy
182 184  
183   -# Copy local host machine files to image
184   -COPY --chown=pyros_user:pyros_user . .
185   -# Copy some aliases
186   -RUN cp .bash_aliases ..
187   -#RUN mv .bash_aliases ..
  185 +RUN mkdir -p ./vendor/guitastro/install/
  186 +RUN mkdir -p ./install/
188 187  
  188 +# Copy local host machine files to image
  189 +# Dockerfile contains this volume => ../..:/home/pyros_user/app
  190 +# Thus, this will copy PYROS_SOFT/* ==TO==> /home/pyros_user/app/ (the WORKDIR)
  191 +COPY ./vendor/guitastro/install/requirements.in ./vendor/guitastro/install/requirements.in
  192 +COPY ./install/requirements.txt ./install/requirements.txt
  193 +COPY ./install/requirements_dev.txt ./install/requirements_dev.txt
189 194  
190 195 # Adding local/bin to path to avoid pip warning
191 196 ENV PATH "$PATH:/home/pyros_user/.local/bin"
192 197  
193 198  
194 199 # Installing click on the image to prevent error on the first execution of the installation script
195   -RUN python3 -m pip install --user click
  200 +RUN python3 -m pip install --user click
196 201  
197   -RUN python3 -m pip install --user setuptools==58
  202 +RUN python3 -m pip install --user setuptools==58
198 203  
199 204 # (EP 23/3/2022) Installing pip-tools for the management of all the requirements*.txt files (python dependencies packages)
200 205 # NB :
... ... @@ -203,14 +208,14 @@ RUN python3 -m pip install --user setuptools==58
203 208 # - Unfortunatly, it is difficult to use with several requirements*.txt files as it is the case for this software : pyros + sphinx + guitastro...
204 209 # - So we cannot yet use it completely and still have to use the traditional "pip install -r" anyway ...
205 210 # - But we can at least use it to generate all the requirements*.txt files in a far better format
206   -RUN python3 -m pip install --user pip-tools
  211 +RUN python3 -m pip install --user pip-tools
207 212  
208 213 # Installing packages required for Guitastro
209 214 RUN pip-compile ./vendor/guitastro/install/requirements.in
210   -RUN python3 -m pip install --user -r ./vendor/guitastro/install/requirements.txt
  215 +RUN python3 -m pip install --user -r ./vendor/guitastro/install/requirements.txt
211 216 # Maybe unnecessary because same requirements as for pyros requirements_dev (?)
212 217 #RUN pip install --user -r ./vendor/guitastro/install/requirements_dev.txt
213 218  
214 219 # Installing packages required for PyROS
215   -RUN python3 -m pip install --user -r ./install/requirements.txt
216   -RUN python3 -m pip install --user -r ./install/requirements_dev.txt
  220 +RUN python3 -m pip install --user -r ./install/requirements.txt
  221 +RUN python3 -m pip install --user -r ./install/requirements_dev.txt
... ...
docker/PYROS_DOCKER_BUILD
  Diff comments   View file @4c0f274
1 1 #!/usr/bin/env bash
2 2  
  3 +# Git clone guitastro
  4 +#
  5 +# This could be done directly in the Dockerfile
  6 +# but it is better to do it here because if it failed during the build
  7 +# we would have to do all the build again...
3 8 [ ! -d "../vendor/guitastro/" ] && git clone https://gitlab.irap.omp.eu/guitastrolib/guitastro.git ../vendor/guitastro/
  9 +[ ! -d "../vendor/guitastro/" ] && echo "Error : git clone of GuitAstro failed => cannot go on with the build" && exit 1
4 10  
5   -export CURRENT_UID=$(id -u)
  11 +export CURRENT_UID=$(id -u)
  12 +export COMPUTER_HOSTNAME=$HOSTNAME
6 13  
7 14 # exit if root
8 15 if [ $CURRENT_UID -eq 0 ] ; then
... ...
docker/PYROS_DOCKER_INSTALL_DB
  Diff comments   View file @4c0f274
1 1 #!/usr/bin/env bash
2 2  
  3 +# PRE-CONDITION : pyros container must be running
  4 +
3 5 # if no container is running
4 6 if ! [ $(docker ps | grep 'pyros' | wc -l) -eq 4 ] ; then
5 7 echo "pyros-db or pyros weren't running, starting them..."
... ...
docker/PYROS_DOCKER_RUN
  Diff comments   View file @4c0f274
1 1 #!/usr/bin/env bash
2 2  
  3 +export CURRENT_UID=$(id -u)
  4 +export COMPUTER_HOSTNAME=$HOSTNAME
  5 +
3 6 # There should be 4 pyros* containers running
4 7 if ! [ $(docker ps | grep 'pyros' | wc -l) -eq 4 ]
5 8 then
... ...
docker/PYROS_DOCKER_RUN_WEBSERVER_ONLY
  Diff comments   View file @4c0f274
1 1 #!/usr/bin/env bash
2 2  
  3 +export CURRENT_UID=$(id -u)
  4 +export COMPUTER_HOSTNAME=$HOSTNAME
  5 +
3 6 #if no container is running
4 7 if ! [ $(docker ps | grep 'pyros' | wc -l) -eq 4 ] ; then
5 8 echo "pyros-db or pyros weren't running, starting them..."
... ...
docker/PYROS_DOCKER_SHELL
  Diff comments   View file @4c0f274
1 1 #!/usr/bin/env bash
2 2  
  3 +export CURRENT_UID=$(id -u)
  4 +export COMPUTER_HOSTNAME=$HOSTNAME
  5 +
3 6 #if no container is running
4 7 if ! [ $(docker ps | grep 'pyros' | wc -l) -eq 4 ]
5 8 then
... ...
docker/PYROS_DOCKER_START
  Diff comments   View file @4c0f274
... ... @@ -12,7 +12,7 @@ docker compose down --remove-orphans
12 12 echo
13 13 echo "Then START:"
14 14 # Take docker-compose.yml folder name ("docker") as containers group name in Docker Desktop by default
15   -docker compose up -d
  15 +docker compose up --no-build -d
16 16 # Change this group name with -p option
17 17 #docker compose -p pyros-app up -d
18 18  
... ...
docker/PYROS_DOCKER_START.bat
  Diff comments   View file @4c0f274
1 1 set CURRENT_UID=1001
2 2 set COMPUTER_HOSTNAME=%COMPUTERNAME%
3 3 docker compose down --remove-orphans
4   -docker compose up -d
  4 +docker compose up --no-build -d
... ...
docker/PYROS_DOCKER_TEST
  Diff comments   View file @4c0f274
1 1 #!/usr/bin/env bash
2 2  
  3 +export CURRENT_UID=$(id -u)
  4 +export COMPUTER_HOSTNAME=$HOSTNAME
  5 +
  6 +
3 7 # if no container is running
4 8 if ! [ $(docker ps | grep 'pyros' | wc -l) -eq 4 ]
5 9 then
... ...
docker/PYROS_DOCKER_UPDATE
  Diff comments   View file @4c0f274
1 1 #!/usr/bin/env bash
2 2  
  3 +# PRE-CONDITION : pyros container must be running
  4 +
3 5 # If no container is running Start it
4 6 if ! [ $(docker ps | grep 'pyros' | wc -l) -eq 4 ]
5 7 then
... ... @@ -16,10 +18,12 @@ echo "Updating Guitastro source code"
16 18 echo "Updating PyROS source code"
17 19 git pull
18 20  
19   -echo "Updating observatory source code"
20   -for dir in ../../PYROS_OBSERVATORY/*
21   -do
22   - git pull
  21 +# 3) Update all observatories with git repo
  22 +for dir in ../../PYROS_OBSERVATORY/* ; do
  23 + if [ -d .git ] ; then
  24 + echo "Updating observatory $dir source code"
  25 + git pull
  26 + fi
23 27 done
24 28  
25 29 # 3) pyros.py update => update BD + doc + Guitastro requirements
... ...
docker/PYROS_DOCKER_UPDATE_AND_RESTART
  Diff comments   View file @4c0f274
1 1 #!/usr/bin/env bash
2 2  
  3 +export CURRENT_UID=$(id -u)
  4 +export COMPUTER_HOSTNAME=$HOSTNAME
  5 +
3 6 abort() {
4 7 echo
5 8 echo "********* !!! ERROR !!! ********"
... ...
docker/docker-compose.yml
  Diff comments   View file @4c0f274
1 1 version: "3.9"
2 2  
  3 +# Application name (group name for the 4 images composing the PyROS app)
3 4 name: pyros-app
4 5  
5 6 services:
... ... @@ -111,6 +112,7 @@ services:
111 112 tty: true
112 113 # stdin_open is the -i option in docker exec
113 114 stdin_open: true
  115 + # Local PYROS_SOFT/ folder is linked to container /home/pyros_user/app folder
114 116 volumes:
115 117 - ../..:/home/pyros_user/app
116 118 # tells which port of local machine can communicate with the docker image (host:container), host is your local machine
... ...
pyros.py
  Diff comments   View file @4c0f274
... ... @@ -115,10 +115,7 @@ try:
115 115 except KeyError:
116 116 WITH_DOCKER = False
117 117  
118   -if type(WITH_DOCKER) is str and re.match("^y$|^Y$|^yes$|^Yes$", WITH_DOCKER.rstrip()) != None:
119   - WITH_DOCKER = True
120   -else:
121   - WITH_DOCKER = False
  118 +WITH_DOCKER = type(WITH_DOCKER) is str and re.match("^y$|^Y$|^yes$|^Yes$", WITH_DOCKER.rstrip()) != None
122 119  
123 120 my_abs_path = os.path.dirname(os.path.realpath(__file__))
124 121 # VENV_ROOT = "private"
... ... @@ -534,7 +531,7 @@ def dbshell():
534 531  
535 532 @pyros_launcher.command(help="Update (only if necessary) the python packages AND the source code AND the DB structure")
536 533 def update():
537   - install_or_update(UPDATE=True)
  534 + _install_or_update(UPDATE=True)
538 535 '''
539 536 print("Running update command")
540 537 # 1) Update source code (git pull)
... ... @@ -568,13 +565,13 @@ def update():
568 565 def install(packages_only, database_only):
569 566 with_packages = not database_only
570 567 with_database = not packages_only
571   - install_or_update(UPDATE=False, with_packages=with_packages,
572   - with_database=with_database)
  568 + _install_or_update(UPDATE=False, with_packages=with_packages, with_database=with_database)
573 569 # install_or_update(UPDATE=False, with_packages=packages_only, with_database=database_only)
574 570 # install_or_update(UPDATE=False, packages_only=packages_only, database_only=database_only)
575 571  
  572 +
576 573 # def install_or_update(UPDATE: bool = False, with_packages: bool = False, with_database: bool = False):
577   -def install_or_update(UPDATE: bool = False, with_packages: bool = True, with_database: bool = True):
  574 +def _install_or_update(UPDATE: bool = False, with_packages: bool = True, with_database: bool = True):
578 575 SQL_USER = os.environ.get("MYSQL_PYROS_LOGIN").strip()
579 576 SQL_PSWD = os.environ.get("MYSQL_PYROS_PWD").strip()
580 577 os.environ["PATH_TO_OBSCONF_FOLDER"] = os.path.join(os.path.abspath(
... ... @@ -583,12 +580,10 @@ def install_or_update(UPDATE: bool = False, with_packages: bool = True, with_dat
583 580 PYROS_DJANGO_BASE_DIR), os.environ["PATH_TO_OBSCONF_FOLDER"] + "observatory.yml")
584 581 ACTION = "UPDATING" if UPDATE else "INSTALLING"
585 582  
586   - # if WITH_DOCKER: with_database = True
  583 + # if WITH_DOCKER: only install the DB (with_database = True)
587 584 if WITH_DOCKER:
588 585 with_packages = False
589   -
590 586 # if not with_packages and not with_database: with_packages = with_database = True
591   -
592 587 print("- with_packages:", with_packages)
593 588 print("- with_database:", with_database)
594 589  
... ... @@ -597,37 +592,36 @@ def install_or_update(UPDATE: bool = False, with_packages: bool = True, with_dat
597 592 # if (os.path.basename(os.getcwd()) != "private"):
598 593  
599 594 num = 0
600   - # 1) Update source code (git pull)
  595 +
601 596 if UPDATE:
  597 + # 1) Update source code (git pull)
602 598 num += 1
603   - printFullTerm(
604   - Colors.BLUE, f"{num}) UPDATING SOURCE CODE: Running git pull")
  599 + printFullTerm(Colors.BLUE, f"{num}) UPDATING SOURCE CODE: Running git pull")
605 600 if not WITH_DOCKER:
606 601 _gitpull() or die()
607 602  
608   - # 2) Update python packages (pip upgrade AND pip install requirements)
609 603 if with_packages:
  604 + # 2) Install venv and packages in it
  605 + # Update python packages (pip upgrade AND pip install requirements)
610 606 num += 1
611 607 printFullTerm(Colors.BLUE, f"{num}) {ACTION} PYTHON PACKAGES")
612 608 # (UPDATE) Re-install VENV if disappeared
613 609 install_venv(EVEN_IF_ALREADY_EXISTS=not UPDATE)
614 610 install_packages()
  611 +
615 612 if UPDATE:
616 613 print("Running UPDATE command")
617 614 else:
618   - # Install GitPython package for git support inside python
  615 + # 3) Install GitPython package for git support inside python
619 616 print("Running INSTALL command")
620 617 try:
621 618 from git import Repo
622 619 except:
623 620 pip = "pip" if IS_WINDOWS else "pip3"
624   - if WITH_DOCKER:
625   - process = subprocess.Popen(
626   - pip + " install --user --upgrade GitPython", shell=True)
627   - else:
628   - process = subprocess.Popen(
629   - pip + " install --upgrade GitPython", shell=True)
  621 + usermode = '--user' if WITH_DOCKER else ''
  622 + process = subprocess.Popen(pip + f" install {usermode} --upgrade GitPython", shell=True)
630 623 process.wait()
  624 + # if run ok
631 625 if process.returncode == 0:
632 626 # self.addExecuted(self.current_command, command)
633 627 from git import Repo
... ... @@ -645,57 +639,58 @@ def install_or_update(UPDATE: bool = False, with_packages: bool = True, with_dat
645 639 change_dir("PREVIOUS")
646 640 '''
647 641  
648   - # Guitastro
649   - GUITASTRO_PATH = os.path.join(os.getcwd(), "./vendor/guitastro")
650   - if with_packages and not WITH_DOCKER :
651   - #GUITASTRO_PATH = os.path.join(os.getcwd(), "../vendor/guitastro")
652   - change_dir("..")
  642 + # 4) Install GUITASTRO + its requirements
  643 + # (ONLY FOR NON DOCKER INSTALL)
653 644 print(os.getcwd())
654   - # 1) clone repo if not yet done
655   - if not os.path.exists(GUITASTRO_PATH) and not WITH_DOCKER :
656   - print("Guitastro : Cloning repository")
657   - cloned_repo = Repo.clone_from(
658   - "https://gitlab.irap.omp.eu/guitastrolib/guitastro.git", GUITASTRO_PATH)
659   - print("Cloned successfully: ", cloned_repo.__class__ is Repo)
660   - if UPDATE and os.path.exists(GUITASTRO_PATH):
661   - gitpull_guitastro()
662   - change_dir("PREVIOUS")
663   - # 2) install/update requirements & generate API doc
664   - if os.path.exists(GUITASTRO_PATH) and not WITH_DOCKER:
665   - # TODO: update guitastro (git pull from vendor/guitastro/)
666   - print("\nGuitastro : Installing/Updating python package dependencies\n")
667   - # Upgrade pip if new version available
668   - os.system(VENV_PYTHON + ' -m pip install --upgrade pip')
669   - # TODO: faire les apt-get intall aussi (indi, ...)
670   - venv_pip_install2(GUITASTRO_PATH + '/install/requirements.txt', '-r')
671   - #venv_pip_install2(GUITASTRO_PATH + '/install/requirements_linux.txt', '-r')
672   - venv_pip_install2(GUITASTRO_PATH + '/install/requirements_dev.txt', '-r')
673   - '''
674   - print("Guitastro : Generating (updating) API documentation (using Sphinx)")
675   - # Make html doc from RST
676   - # cd doc/sourcedoc/
677   - change_dir(GUITASTRO_PATH + '/doc_rst/')
678   - # ./MAKE_DOC.sh
679   - res = execProcess('/bin/bash MAKE_DOC.sh')
680   - # Come back to where we were before
681   - # cd -
682   - change_dir("PREVIOUS")
683   - '''
684   -
685   -
686   - # 3) Update PlantUML diagrams
  645 + if not WITH_DOCKER:
  646 + GUITASTRO_PATH = os.path.join(os.getcwd(), "./vendor/guitastro")
  647 + if with_packages:
  648 + change_dir("..")
  649 + print(os.getcwd())
  650 + if not os.path.exists(GUITASTRO_PATH):
  651 + # a) clone repo if not yet done
  652 + print("Guitastro : Cloning repository")
  653 + cloned_repo = Repo.clone_from(
  654 + "https://gitlab.irap.omp.eu/guitastrolib/guitastro.git", GUITASTRO_PATH)
  655 + print("Cloned successfully: ", cloned_repo.__class__ is Repo)
  656 + if os.path.exists(GUITASTRO_PATH):
  657 + if UPDATE:
  658 + gitpull_guitastro()
  659 + change_dir("PREVIOUS")
  660 + # b) install/update requirements & generate API doc
  661 + # TODO: update guitastro (git pull from vendor/guitastro/)
  662 + print("\nGuitastro : Installing/Updating python package dependencies\n")
  663 + # Upgrade pip if new version available
  664 + os.system(VENV_PYTHON + ' -m pip install --upgrade pip')
  665 + # TODO: faire les apt-get intall aussi (indi, ...)
  666 + _venv_pip_install2(GUITASTRO_PATH + '/install/requirements.txt', '-r')
  667 + #venv_pip_install2(GUITASTRO_PATH + '/install/requirements_linux.txt', '-r')
  668 + _venv_pip_install2(GUITASTRO_PATH + '/install/requirements_dev.txt', '-r')
  669 + '''
  670 + print("Guitastro : Generating (updating) API documentation (using Sphinx)")
  671 + # Make html doc from RST
  672 + # cd doc/sourcedoc/
  673 + change_dir(GUITASTRO_PATH + '/doc_rst/')
  674 + # ./MAKE_DOC.sh
  675 + res = execProcess('/bin/bash MAKE_DOC.sh')
  676 + # Come back to where we were before
  677 + # cd -
  678 + change_dir("PREVIOUS")
  679 + '''
  680 +
  681 + # 5) Update PlantUML diagrams (in UPDATE or INSTALL mode)
687 682 num += 1
688 683 printFullTerm(Colors.BLUE, f"{num}) UPDATING UML DIAGRAMS")
689 684 _update_plantuml_diags() or die()
690 685 print(os.getcwd())
691 686  
692   - # 4) Update Sphinx API doc
  687 + # 6) Update Sphinx API doc (Non Windows only) (in UPDATE or INSTALL mode)
693 688 num += 1
694 689 printFullTerm(Colors.BLUE, f"{num}) UPDATING API DOC (with Sphinx)")
695 690 if not IS_WINDOWS:
696 691 _update_api_doc() or die()
697 692  
698   - # 5) Install/Update database structure (make migrations + migrate)
  693 + # 7) Install/Update database structure (make migrations + migrate)
699 694 if with_database:
700 695 num += 1
701 696 printFullTerm(Colors.BLUE, f"{num}) {ACTION} DATABASE")
... ... @@ -703,9 +698,8 @@ def install_or_update(UPDATE: bool = False, with_packages: bool = True, with_dat
703 698 _updatedb() or die()
704 699 else:
705 700 res = install_database(VENV)
706   - if(res == 0):
707   - print(
708   - f"You can connect to PyROS as '{SQL_USER}' user with the password '{SQL_PSWD}' ")
  701 + if (res == 0):
  702 + print(f"You can connect to PyROS as '{SQL_USER}' user with the password '{SQL_PSWD}' ")
709 703 return True
710 704  
711 705  
... ... @@ -727,8 +721,7 @@ def test(module, function):
727 721 if module is None:
728 722 # apps = ['obs_config','scp_mgmt','common', 'scheduling', 'seq_submit', 'user_mgmt', 'alert_mgmt.tests.TestStrategyChange']
729 723 # Removing alert_mgmt, scheduler from tests
730   - apps = ['obs_config', "scp_mgmt",
731   - 'dashboard', 'user_mgmt', 'seq_submit','api']
  724 + apps = ['obs_config', "scp_mgmt", 'dashboard', 'user_mgmt', 'seq_submit', 'api']
732 725 else:
733 726 os.environ["PATH_TO_OBSCONF_FILE"] = os.path.join(os.path.abspath(
734 727 PYROS_DJANGO_BASE_DIR), "obs_config/fixtures/observatory_configuration_ok_simple.yml")
... ... @@ -760,7 +753,7 @@ def test(module, function):
760 753 # execProcessFromVenv('manage.py test ' + app) or die()
761 754 # KEEP test_pyros database after tests
762 755 # execProcessFromVenv('manage.py test --keep ' + app) or die()
763   - execProcessFromVenv('manage.py test --keep --noinput ' + app,foreground=True) or die()
  756 + execProcessFromVenv('manage.py test --keep --noinput ' + app, foreground=True) or die()
764 757 change_dir("PREVIOUS")
765 758 # execProcess("python install.py install")
766 759 return True
... ... @@ -789,8 +782,7 @@ def gitpull_guitastro():
789 782 change_dir("./vendor/guitastro/")
790 783 GIT = "git.exe" if IS_WINDOWS else "git"
791 784 if not test_mode():
792   - return execProcess(f"{GIT} pull",foreground=True)
793   -
  785 + return execProcess(f"{GIT} pull", foreground=True)
794 786 return True
795 787  
796 788  
... ... @@ -816,21 +808,24 @@ def initdb():
816 808 return True
817 809  
818 810  
819   -
820   -@pyros_launcher.command(help="Install dependencies")
  811 +@pyros_launcher.command(help="Install dependencies from Observatory config DEPENDENCY section")
821 812 # @global_test_options
822   -@click.option('--observatory', '-o', help='the observatory name to be used')
823   -@click.option('--unit', '-u', help='the unit name to be used')
  813 +@click.option('--observatory', '-o', help='the Observatory name to be used')
  814 +@click.option('--unit', '-u', help='the Unit name to be used')
824 815 @click.option("--foreground","-fg", is_flag=True, help="Print stdout and error in terminal")
825 816 def install_dependencies(observatory: str, unit: str, foreground: bool):
826   - if observatory == None or len(observatory) == 0:
  817 + if (observatory is None) or (len(observatory) == 0):
827 818 observatory = "default"
828 819 default_obsconfig_folder = os.path.join(
829   - os.path.abspath(PYROS_DJANGO_BASE_DIR), "../../../config/pyros_observatory/")
  820 + os.path.abspath(PYROS_DJANGO_BASE_DIR),
  821 + "../../../config/pyros_observatory/"
  822 + )
830 823 path_to_obs_config_folder = default_obsconfig_folder+"pyros_observatory_"+observatory+"/"
831 824 else:
832 825 observatories_configuration_folder = os.path.join(
833   - os.path.abspath(PYROS_DJANGO_BASE_DIR), "../../../../PYROS_OBSERVATORY/")
  826 + os.path.abspath(PYROS_DJANGO_BASE_DIR),
  827 + "../../../../PYROS_OBSERVATORY/"
  828 + )
834 829 if len(glob.glob(observatories_configuration_folder+"pyros_observatory_"+observatory+"/")) != 1:
835 830 # Observatory configuration folder not found
836 831 print(
... ... @@ -845,15 +840,14 @@ def install_dependencies(observatory: str, unit: str, foreground: bool):
845 840 # Search for observatory config file
846 841 obs_config_file_name = glob.glob(os.path.join(path_to_obs_config_folder, "observatory.yml"))[0]
847 842  
848   - obs_config_file_path = os.path.join(
849   - path_to_obs_config_folder, obs_config_file_name)
  843 + obs_config_file_path = os.path.join(path_to_obs_config_folder, obs_config_file_name)
850 844 os.environ["PATH_TO_OBSCONF_FILE"] = obs_config_file_path
851 845 os.environ["PATH_TO_OBSCONF_FOLDER"] = path_to_obs_config_folder
852 846 os.environ["unit_name"] = unit if unit else ''
853 847  
854 848 # add path to pyros_django folder as the config class is supposed to work within this folder
855 849 cmd_test_obs_config = f"-c \"from src.core.pyros_django.obs_config.obsconfig_class import OBSConfig\nOBSConfig('{obs_config_file_path}')\""
856   - if not execProcessFromVenv(cmd_test_obs_config,foreground=True):
  850 + if not execProcessFromVenv(cmd_test_obs_config, foreground=True):
857 851 # Observatory configuration has an issue
858 852 exit(1)
859 853 else:
... ... @@ -862,12 +856,8 @@ def install_dependencies(observatory: str, unit: str, foreground: bool):
862 856 from git import Repo, GitError
863 857 except:
864 858 pip = "pip" if IS_WINDOWS else "pip3"
865   - if WITH_DOCKER:
866   - process = subprocess.Popen(
867   - pip + " install --user --upgrade GitPython", shell=True)
868   - else:
869   - process = subprocess.Popen(
870   - pip + " install --upgrade GitPython", shell=True)
  859 + usermode = '--user' if WITH_DOCKER else ''
  860 + process = subprocess.Popen(pip + f" install {usermode} --upgrade GitPython", shell=True)
871 861 process.wait()
872 862 if process.returncode == 0:
873 863 # self.addExecuted(self.current_command, command)
... ... @@ -1129,8 +1119,7 @@ def new_start(configfile: str, observatory: str, unit: str, computer_hostname: s
1129 1119 # pip = "pip" if platform.system() == "Windows" else "pip3"
1130 1120 # TODO: "python -m pip" au lieu de "pip"
1131 1121 pip = "pip" if IS_WINDOWS else "pip3"
1132   - process = subprocess.Popen(
1133   - pip + " install --user --upgrade pykwalify", shell=True)
  1122 + process = subprocess.Popen(pip + " install --user --upgrade pykwalify", shell=True)
1134 1123 process.wait()
1135 1124 if process.returncode == 0:
1136 1125 # self.addExecuted(self.current_command, command)
... ... @@ -1311,12 +1300,9 @@ def _update_api_doc():
1311 1300 print(os.getcwd())
1312 1301 DOC_RST_PATH = "doc/doc_rst/"
1313 1302 # 0) Upgrade pip if new version available
1314   - if WITH_DOCKER:
1315   - os.system(VENV_PYTHON + ' -m pip install --user --upgrade pip')
1316   - else:
1317   - os.system(VENV_PYTHON + ' -m pip install --upgrade pip')
  1303 + _venv_pip_install2('pip', '--upgrade')
1318 1304 # 1) install/update Sphinx requirements (only if not yet installed)
1319   - venv_pip_install2(DOC_RST_PATH+'requirements.txt', '-r')
  1305 + _venv_pip_install2(DOC_RST_PATH+'requirements.txt', '-r')
1320 1306 # 2) make html doc from RST doc
1321 1307 # cd doc/sourcedoc/
1322 1308 change_dir(DOC_RST_PATH)
... ... @@ -1488,20 +1474,24 @@ def notused_install_required():
1488 1474 exit(1)
1489 1475  
1490 1476  
1491   -def venv_pip_install(package_name: str, options: str = ''):
1492   - if WITH_DOCKER:
1493   - os.system(VENV_PIP + ' install --user ' + options + ' ' + package_name)
1494   - else:
1495   - os.system(VENV_PIP + ' install ' + options + ' ' + package_name)
  1477 +def _venv_pip_install_from_venv(PIP_MODULE: bool, package_name: str, options: str = ''):
  1478 + '''
  1479 + pip install package_name with options
  1480 + If PIP_MODULE install with 'python -m pip' else install with 'pip'
  1481 + If WITH_DOCKER => install in user mode (with --user option)
  1482 + '''
  1483 + #_pip_install(options + ' ' + package_name)
  1484 + pip_cmd = VENV_PYTHON+' -m pip' if PIP_MODULE else VENV_PIP
  1485 + usermode = '--user' if WITH_DOCKER else ''
  1486 + os.system(f'{pip_cmd} install {usermode} {options} {package_name}')
1496 1487  
1497 1488  
1498   -def venv_pip_install2(package_name: str, options: str = ''):
1499   - if WITH_DOCKER:
1500   - os.system(VENV_PYTHON + ' -m pip install --user '
1501   - + options + ' ' + package_name)
1502   - else:
1503   - os.system(VENV_PYTHON + ' -m pip install '
1504   - + options + ' ' + package_name)
  1489 +def _venv_pip_install(package_name: str, options: str = ''):
  1490 + _venv_pip_install_from_venv(False, package_name, options)
  1491 +
  1492 +
  1493 +def _venv_pip_install2(package_name: str, options: str = ''):
  1494 + _venv_pip_install_from_venv(True, package_name, options)
1505 1495  
1506 1496  
1507 1497 def install_venv(EVEN_IF_ALREADY_EXISTS: bool = False):
... ... @@ -1573,10 +1563,7 @@ def install_packages():
1573 1563 print(Colors.LOG_BLUE + "-----------------------------Upgrade pip, wheel, and setuptools" +
1574 1564 "-----------------------------"+END_OF_LINE + Colors.END)
1575 1565 # Upgrade pip
1576   - if WITH_DOCKER:
1577   - os.system(VENV_PYTHON + ' -m pip install --user --upgrade pip')
1578   - else:
1579   - os.system(VENV_PYTHON + ' -m pip install --upgrade pip')
  1566 + _venv_pip_install2('pip', '--upgrade')
1580 1567 '''
1581 1568 if (platform.system() == "Windows"):
1582 1569 os.system(venv + '\Scripts\python -m pip install --upgrade pip')
... ... @@ -1585,27 +1572,24 @@ def install_packages():
1585 1572 '''
1586 1573  
1587 1574 # Pip upgrade wheel and setuptools
1588   - venv_pip_install('wheel', '--upgrade')
  1575 + _venv_pip_install('wheel', '--upgrade')
1589 1576 # os.system(VENV_PIP+' install --upgrade wheel')
1590 1577 # Not working with python 3.8 (on 17/02/2022)
1591 1578 # venv_pip_install('setuptools', '--upgrade')
1592   - if WITH_DOCKER:
1593   - os.system(VENV_PIP+' install --user setuptools==58')
1594   - else:
1595   - os.system(VENV_PIP+' install setuptools==58')
  1579 + _venv_pip_install('setuptools==58')
1596 1580  
1597 1581 # Pip install required packages from REQUIREMENTS file
1598 1582 print()
1599 1583  
1600 1584 # General normal packages
1601 1585 print(Colors.LOG_BLUE + "-----------------------------Installing python packages via pip-----------------------------" + Colors.END)
1602   - venv_pip_install('../install/'+REQUIREMENTS, '-r')
  1586 + _venv_pip_install('../install/'+REQUIREMENTS, '-r')
1603 1587 # os.system(VENV_PIP+' install -r ../install' + os.sep + REQUIREMENTS)
1604 1588  
1605 1589 # DEV only packages
1606 1590 if DEV:
1607 1591 print(Colors.LOG_BLUE + "-----------------------------Installing DEV python packages via pip-----------------------------" + Colors.END)
1608   - venv_pip_install('../install/'+REQUIREMENTS_DEV, '-r')
  1592 + _venv_pip_install('../install/'+REQUIREMENTS_DEV, '-r')
1609 1593 print("FIN INSTALL PACKAGES")
1610 1594 return None
1611 1595  
... ... @@ -1662,12 +1646,9 @@ def install_observatory(observatory):
1662 1646 from git import Repo, GitError
1663 1647 except:
1664 1648 pip = "pip" if IS_WINDOWS else "pip3"
1665   - if WITH_DOCKER:
1666   - process = subprocess.Popen(
1667   - pip + " install --user --upgrade GitPython", shell=True)
1668   - else:
1669   - process = subprocess.Popen(
1670   - pip + " install --upgrade GitPython", shell=True)
  1649 +
  1650 + usermode = '--user' if WITH_DOCKER else ''
  1651 + process = subprocess.Popen(pip + f" install {usermode} --upgrade GitPython", shell=True)
1671 1652 process.wait()
1672 1653 if process.returncode == 0:
1673 1654 # self.addExecuted(self.current_command, command)
... ...
src/core/pyros_django/majordome/agent/AgentBasic.py renamed to src/core/pyros_django/majordome/agent/A_Basic.py
  Diff comments   View file @4c0f274
1 1 #!/usr/bin/env python3
2 2  
3 3  
4   -import time
5   -import sys, argparse
  4 +#import time
  5 +import sys
  6 +import argparse
  7 +import os
6 8  
7 9 ##import utils.Logger as L
8 10  
9 11 ##from .Agent import Agent
10 12 ##sys.path.append("..")
11 13 ###from agent.Agent import Agent, build_agent
12   -sys.path.append("../../../..")
  14 +
  15 +###sys.path.append("../../../..")
  16 +
  17 +# pwd = PYROS/
  18 +pwd = os.environ['PROJECT_ROOT_PATH']
  19 +if pwd not in sys.path:
  20 + sys.path.append(pwd)
  21 +
  22 +# Add paths PYROS/src and PYROS/src/core/pyros_django
  23 +short_paths = ['src', 'src/core/pyros_django']
  24 +for short_path in short_paths:
  25 + path = os.path.join(pwd, short_path)
  26 + # ou plutot ?
  27 + #path = os.path.abspath(os.path.join(pwd, short_path))
  28 + if path not in sys.path:
  29 + sys.path.insert(0, path)
  30 +
13 31 #from src.core.pyros_django.common.models import AgentCmd
14   -from src.core.pyros_django.majordome.agent.Agent import Agent, build_agent
  32 +#from src.core.pyros_django.majordome.agent.Agent import Agent, build_agent
  33 +from majordome.agent.Agent import Agent, build_agent
  34 +
15 35 #from src.core.pyros_django.common.models import AgentCmd
16 36  
17 37 from typing import List, Tuple, Union, Any
... ... @@ -20,16 +40,16 @@ from typing import List, Tuple, Union, Any
20 40  
21 41  
22 42  
23   -class AgentBasic(Agent):
  43 +class A_Basic(Agent):
24 44  
25 45 # FOR TEST ONLY
26   - # Run this agent in simulator mode
  46 + # - Run this agent in simulator mode
27 47 TEST_MODE = False
28   - # Run the assertion tests at the end
  48 + # - Run the assertion tests at the end
29 49 TEST_WITH_FINAL_TEST = False
30 50 #TEST_MAX_DURATION_SEC = None
31 51 TEST_MAX_DURATION_SEC = 400
32   - # Who should I send commands to ?
  52 + # - Who should I send commands to ?
33 53 TEST_COMMANDS_DEST = "myself"
34 54 #TEST_COMMANDS_DEST = "AgentB"
35 55 # Scenario to be executed
... ... @@ -459,7 +479,7 @@ if __name__ == "__main__":
459 479 parser.add_argument("--computer",dest="computer",help='Launch agent with simulated computer hostname',action="store")
460 480 parser.add_argument("-t", action="store_true")
461 481 args = vars(parser.parse_args())
462   - agent = build_agent(AgentBasic,param_constr=args)
  482 + agent = build_agent(A_Basic, param_constr=args)
463 483  
464 484 '''
465 485 TEST_MODE, configfile = extract_parameters()
... ...
src/core/pyros_django/majordome/agent/Agent.py
  Diff comments   View file @4c0f274
... ... @@ -154,12 +154,13 @@ import config.old_config as config_old
154 154 #from config import *
155 155  
156 156 from majordome.models import AgentSurvey, AgentCmd, AgentLogs
157   -from user_mgmt.models import Period
  157 +from user_mgmt.models import Period, Quota
158 158  
159 159 from vendor.guitastro.src.guitastro import Ephemeris
160 160 import pickle
161 161  
162 162  
  163 +# Aliases for Cmd exceptions
163 164 CmdException = AgentCmd.CmdException
164 165 CmdExceptionUnknown = AgentCmd.CmdExceptionUnknown
165 166 CmdExceptionUnimplemented = AgentCmd.CmdExceptionUnimplemented
... ... @@ -218,7 +219,7 @@ class Colors:
218 219 BOLD = "\033[1m"
219 220 UNDERLINE = "\033[4m"
220 221  
221   -def printColor(color: Colors, message, file=sys.stdout, eol=os.linesep, forced=False):
  222 +def print_colored(color: Colors, message, file=sys.stdout, eol=os.linesep, forced=False):
222 223 #system = platform.system()
223 224 """
224 225 if (self.disp == False and forced == False):
... ... @@ -237,10 +238,10 @@ def printFullTerm(color: Colors, string: str):
237 238 row = 1000
238 239 disp = True
239 240 value = int(columns / 2 - len(string) / 2)
240   - printColor(color, "-" * value, eol="")
241   - printColor(color, string, eol="")
  241 + print_colored(color, "-" * value, eol="")
  242 + print_colored(color, string, eol="")
242 243 value += len(string)
243   - printColor(color, "-" * (columns - value))
  244 + print_colored(color, "-" * (columns - value))
244 245 return 0
245 246  
246 247  
... ... @@ -301,23 +302,22 @@ class Agent:
301 302 # --- - INSTANCE attributes are accessible via agent.__dict__
302 303 # ---
303 304  
304   -
305 305 class EXEC_MODE(Enum):
306 306 SEQUENTIAL = 0
307 307 THREAD = 1
308 308 PROCESS = 2
309 309  
310 310 # Default modes
311   - DEBUG_MODE:bool = False
  311 + DEBUG_MODE: bool = False
312 312 #TEST_MODE = False
313 313  
314 314 # By default, a command is valid during 5s (and then perempted)
315   - DEFAULT_CMD_VALIDITY_DURATION:int = 5
  315 + DEFAULT_CMD_VALIDITY_DURATION: int = 5
316 316  
317 317 # Wait a fixed number of seconds before each loop ?
318 318 #WITH_RANDOM_WAIT = False
319 319 # 1 sec by default
320   - __DELAY_NB_SEC:int = 1
  320 + __DELAY_NB_SEC: int = 1
321 321 # - YES if TEST mode (in init())
322 322  
323 323 # Default LOG level is INFO
... ... @@ -332,7 +332,7 @@ class Agent:
332 332 - exec_mode (EXEC_MODE) : EXEC_MODE.SEQUENTIAL, EXEC_MODE.THREAD, or EXEC_MODE.PROCESS
333 333 - tooltip : description text (displayed on clic)
334 334 '''
335   - _AGENT_SPECIFIC_COMMANDS: Dict [ str, Tuple[int, EXEC_MODE, str] ] = {
  335 + _AGENT_SPECIFIC_COMMANDS: Dict[str, Tuple[int, EXEC_MODE, str]] = {
336 336 # Format : “cmd_name” : (timeout, exec_mode, tooltip)
337 337  
338 338 #"do_specific1" : (10, EXEC_MODE.SEQUENTIAL, ''),
... ... @@ -352,14 +352,15 @@ class Agent:
352 352 # Maximum duration of this agent (only for SIMULATION mode)
353 353 # If set to 0, it will never exit except if asked (or CTRL-C)
354 354 # If set to 20, it will exit after 20s
355   - TEST_MAX_DURATION_SEC :int=0
  355 + TEST_MAX_DURATION_SEC: int = 0
356 356 #TEST_MAX_DURATION_SEC = 30
357 357 # Run this agent in simulator mode
358 358 #TEST_MODE = True
359   - WITH_SIMULATOR:bool = False
  359 + WITH_SIMULATOR: bool = False
360 360 # Run the assertion tests at the end
361   - TEST_WITH_FINAL_TEST:bool = False
  361 + TEST_WITH_FINAL_TEST: bool = False
362 362  
  363 + # Aliases
363 364 CMD_STATUS = AgentCmd.CMD_STATUS_CODES
364 365 AGT_STATUS = AgentSurvey.STATUS_CHOICES
365 366  
... ... @@ -393,8 +394,8 @@ class Agent:
393 394 ##_TEST_COMMANDS_LIST: List[ Tuple[ bool, str, int, Optional[str], AgentCmd.CMD_STATUS_CODES ] ] = [
394 395  
395 396 # Alias type for _TEST_COMMANDS_LIST (for more readability)
396   - TestCommand = Tuple[ bool, str, Optional[int], Optional[str], Optional[int]]
397   - _TEST_COMMANDS_LIST: List[ TestCommand ] = [
  397 + TestCommand = Tuple[bool, str, Optional[int], Optional[str], Optional[int]]
  398 + _TEST_COMMANDS_LIST: List[TestCommand] = [
398 399 # Format : (DO_IT, "self cmd_name cmd_args", validity, "expected_result", expected_status),
399 400  
400 401 #("self do_stop now", 200, '15.5', None),
... ... @@ -591,8 +592,8 @@ class Agent:
591 592 ##_cmdts: AgentCmd = None
592 593 ##_next_cmdts = None
593 594  
594   - __agent_survey:AgentSurvey = None
595   - __pending_commands:QuerySet = None # []
  595 + __agent_survey: AgentSurvey = None
  596 + __pending_commands: QuerySet = None # []
596 597  
597 598 # List of agents I will send commands to
598 599 _my_client_agents_aliases = []
... ... @@ -610,28 +611,28 @@ class Agent:
610 611  
611 612 # new obsconfig init for agent:
612 613 ##def __init__(self, RUN_IN_THREAD=True):
613   - def __init__(self,simulated_computer=None):
  614 + def __init__(self, simulated_computer=None):
614 615  
615 616 # Instance attributes declaration (with default values, or None)
616   - self.__UP_SINCE : Final = datetime.now(tz=timezone.utc)
  617 + self.__UP_SINCE: Final = datetime.now(tz=timezone.utc)
617 618 #self.UP_SINCE = datetime.utcnow()
618   - self.__ROUTINE_ITER_START_IS_RUNNING:bool = False
619   - self.__ROUTINE_ITER_END_IS_RUNNING:bool = False
620   - self.__test_cmd_received_num:int = 0 # only for tests
  619 + self.__ROUTINE_ITER_START_IS_RUNNING: bool = False
  620 + self.__ROUTINE_ITER_END_IS_RUNNING: bool = False
  621 + self.__test_cmd_received_num: int = 0 # only for tests
621 622 # Current Command running
622   - self.__CC :Optional[AgentCmd] #= None
623   - self.__CC_thread :Union[StoppableThreadEvenWhenSleeping, multiprocessing.Process] #= None
  623 + self.__CC: Optional[AgentCmd] # = None
  624 + self.__CC_thread: Union[StoppableThreadEvenWhenSleeping, multiprocessing.Process] #= None
624 625 # Previous Command running
625 626 ##self.__CC_prev :Optional[AgentCmd] = None
626 627 # Current Command exception (if occurs)
627   - self.__CCE :Optional[Exception] #= None
  628 + self.__CCE: Optional[Exception] #= None
628 629 self.name = "Generic Agent"
629   - self.__status :str #= None
630   - self.__mode :str #= None
631   - self.unit :str #= None
  630 + self.__status: str # = None
  631 + self.__mode: str # = None
  632 + self.unit: str # = None
632 633 #self.TEST_COMMANDS :List #= None
633   - self.TEST_COMMANDS :Iterable[Agent.TestCommand] #= None
634   - self.__iter_num :int = 0
  634 + self.TEST_COMMANDS: Iterable[Agent.TestCommand] # = None
  635 + self.__iter_num: int = 0
635 636 #print(AgentSurvey.MODE_CHOICES.IDLE)
636 637 #sys.exit()
637 638  
... ... @@ -639,18 +640,21 @@ class Agent:
639 640 #self.__mode = self.MODE_ATTENTIVE
640 641 self.set_mode_attentive()
641 642 #self._set_mode(MODES.)
  643 +
  644 + # Set Obs config
642 645 obs_config_file_path = os.environ["PATH_TO_OBSCONF_FILE"]
643 646 path_to_obs_config_folder = os.environ["PATH_TO_OBSCONF_FOLDER"]
644 647 unit = os.environ["unit_name"]
645   - oc = OBSConfig(obs_config_file_path,unit)
  648 + oc = OBSConfig(obs_config_file_path, unit)
646 649 pyros_yaml_path = os.environ["pyros_config_file"]
647 650 pyros_config = ConfigPyros(pyros_yaml_path)
648 651 self.set_config(oc, obs_config_file_path, path_to_obs_config_folder, unit, pyros_config, pyros_yaml_path)
  652 +
  653 + # Agent name
649 654 agent_name_from_config = self.get_config().get_agent_name_from_config(self.__class__.__name__,simulated_computer)
650   - if agent_name_from_config:
651   - self.name = agent_name_from_config
652   - else:
653   - self.name = self.__class__.__name__
  655 + self.name = agent_name_from_config if agent_name_from_config else self.__class__.__name__
  656 +
  657 + # LOG
654 658 log.addHandler(handler_filebyagent(logging.INFO, self.name))
655 659 #log.addHandler(handler_filebyagent(logging.INFO, self.name))
656 660 log.debug("start Agent init")
... ... @@ -670,10 +674,11 @@ class Agent:
670 674  
671 675 self.TEST_COMMANDS = iter(self._TEST_COMMANDS_LIST)
672 676 ##self.RUN_IN_THREAD = RUN_IN_THREAD
  677 +
673 678 self.__set_status(self.AGT_STATUS.LAUNCHED)
674 679 ####self._set_idle()
675 680  
676   - # Create 1st survey if none
  681 + # Get survey or create 1st one if none
677 682 #tmp = AgentSurvey.objects.filter(name=self.name)
678 683 #if len(tmp) == 0:
679 684 #nb_agents = AgentSurvey.objects.filter(name=self.name).count()
... ... @@ -857,18 +862,16 @@ class Agent:
857 862 @property
858 863 def ROUTINE_ITER_START_IS_RUNNING(self):
859 864 return self.__ROUTINE_ITER_START_IS_RUNNING
  865 +
860 866 @property
861 867 def ROUTINE_ITER_END_IS_RUNNING(self):
862 868 return self.__ROUTINE_ITER_END_IS_RUNNING
863 869  
864   -
865   -
866   -
867 870 def set_config(self, oc: OBSConfig, obs_config_file_path: str, path_to_obs_config_folder: str, unit: str, pyros_config: str, pyros_yaml_path: str):
868 871 self._oc = {
869   - 'config' : oc,
870   - 'pyros_config' : pyros_config,
871   - 'env' : [
  872 + 'config': oc,
  873 + 'pyros_config': pyros_config,
  874 + 'env': [
872 875 obs_config_file_path,
873 876 path_to_obs_config_folder,
874 877 unit,
... ... @@ -962,10 +965,6 @@ class Agent:
962 965 print("\n")
963 966  
964 967  
965   -
966   -
967   -
968   -
969 968 def get_config_filename(self, config_filename: str):
970 969 if not config_filename:
971 970 #config_filename = self.DEFAULT_CONFIG_FILE_NAME
... ... @@ -1013,7 +1012,7 @@ class Agent:
1013 1012  
1014 1013  
1015 1014  
1016   - def _get_real_agent_name(self, agent_alias_name:str)->str:
  1015 + def _get_real_agent_name(self, agent_alias_name: str) -> str:
1017 1016 #self.printd("key is", agent_alias_name)
1018 1017 '''
1019 1018 if not self._my_client_agents: return agent_alias_name
... ... @@ -1023,7 +1022,7 @@ class Agent:
1023 1022  
1024 1023  
1025 1024  
1026   - def run(self, nb_iter:int=None, FOR_REAL:bool=True):
  1025 + def run(self, nb_iter: int = None, FOR_REAL: bool = True):
1027 1026 """
1028 1027 FOR_REAL: set to False if you don't want Agent to send commands to devices but just print messages without really doing anything
1029 1028 """
... ... @@ -3593,7 +3592,17 @@ class Agent:
3593 3592 val[0] = d_total
3594 3593 val[-1] = d_total - val[-2]
3595 3594 # --- update db TODO
3596   -
  3595 + quota = Quota()
  3596 + quota_attributes = {}
  3597 + quota_attributes["id_period"] = operiod.id
  3598 + quota_attributes["night_id"] = night
  3599 + quota_attributes["d_previousq"] = d_prev
  3600 + quota_attributes["d_currentq"] = d_cur
  3601 + quota_attributes["d_totalq"] = d_total
  3602 + quota_attributes["d_nextq"] = d_total - d_cur
  3603 + quota.set_attributes_and_save(quota_attributes)
  3604 + operiod.quota = quota
  3605 + operiod.save()
3597 3606 #log.info(f"Write {filename=}")
3598 3607 #pickle.dump(night_info, open(filename, "wb"))
3599 3608 return night_info
... ...
src/core/pyros_django/misc/fixtures/initial_fixture_dev_TZ.json
  Diff comments   View file @4c0f274
... ... @@ -287,8 +287,7 @@
287 287 "fields": {
288 288 "name": "Admin",
289 289 "desc": "",
290   - "priority": 8,
291   - "quota": 9999.0
  290 + "priority": 8
292 291 }
293 292 },
294 293 {
... ... @@ -297,8 +296,7 @@
297 296 "fields": {
298 297 "name": "Observer",
299 298 "desc": "",
300   - "priority": 2,
301   - "quota": 9999.0
  299 + "priority": 2
302 300 }
303 301 },
304 302 {
... ... @@ -307,8 +305,7 @@
307 305 "fields": {
308 306 "name": "TAC",
309 307 "desc": "",
310   - "priority": 1,
311   - "quota": 9999.0
  308 + "priority": 1
312 309 }
313 310 },
314 311 {
... ... @@ -317,8 +314,7 @@
317 314 "fields": {
318 315 "name": "Management board member",
319 316 "desc": "",
320   - "priority": 3,
321   - "quota": 9999.0
  317 + "priority": 3
322 318 }
323 319 },
324 320 {
... ... @@ -327,8 +323,7 @@
327 323 "fields": {
328 324 "name": "Operator",
329 325 "desc": "",
330   - "priority": 4,
331   - "quota": 9999.0
  326 + "priority": 4
332 327 }
333 328 },
334 329 {
... ... @@ -337,8 +332,7 @@
337 332 "fields": {
338 333 "name": "Unit-PI",
339 334 "desc": "",
340   - "priority": 7,
341   - "quota": 9999.0
  335 + "priority": 7
342 336 }
343 337 },
344 338 {
... ... @@ -347,8 +341,7 @@
347 341 "fields": {
348 342 "name": "Unit-board",
349 343 "desc": "",
350   - "priority": 6,
351   - "quota": 9999.0
  344 + "priority": 6
352 345 }
353 346 },
354 347 {
... ... @@ -357,8 +350,7 @@
357 350 "fields": {
358 351 "name": "Visitor",
359 352 "desc": "Account without any privilege",
360   - "priority": 0,
361   - "quota": 0.0
  353 + "priority": 0
362 354 }
363 355 },
364 356 {
... ... @@ -366,7 +358,7 @@
366 358 "pk": 2,
367 359 "fields": {
368 360 "name": "CNRS",
369   - "quota": 20
  361 + "f_quota": 0.2
370 362 }
371 363 },
372 364 {
... ... @@ -374,7 +366,7 @@
374 366 "pk": 1,
375 367 "fields": {
376 368 "name": "CNES",
377   - "quota": 80
  369 + "f_quota": 0.8
378 370 }
379 371 },
380 372 {
... ...
src/core/pyros_django/scheduling/A_Scheduler.py
  Diff comments   View file @4c0f274
... ... @@ -29,6 +29,7 @@ import argparse
29 29 import os
30 30 import pickle
31 31 import socket
  32 +
32 33 pwd = os.environ['PROJECT_ROOT_PATH']
33 34 if pwd not in sys.path:
34 35 sys.path.append(pwd)
... ... @@ -39,7 +40,8 @@ for short_path in short_paths:
39 40 if path not in sys.path:
40 41 sys.path.insert(0, path)
41 42  
42   -from src.core.pyros_django.majordome.agent.Agent import Agent, build_agent, log, parse_args
  43 +#from src.core.pyros_django.majordome.agent.Agent import Agent, build_agent, log, parse_args
  44 +from majordome.agent.Agent import Agent, build_agent, log, parse_args
43 45 from seq_submit.models import Sequence
44 46 from user_mgmt.models import Period, ScientificProgram, SP_Period
45 47 from scheduling.models import PredictiveSchedule, EffectiveSchedule
... ... @@ -220,6 +222,12 @@ class A_Scheduler(Agent):
220 222 # 'vote_referee2'
221 223 """
222 224  
  225 +
  226 + def update_db_quota_sequence(sequence, quota_attributes, id_period, night_id, d_total=sequence_info['duration']):
  227 + sequence_quota = sequence.quota
  228 + sp_quota = sequence.scientific_program
  229 + institute_quota =
  230 +
223 231 def _compute_schedule_1(self):
224 232 """Simple scheduler based on selection-insertion one state algorithm.
225 233  
... ...
src/core/pyros_django/scp_mgmt/A_SCP_Manager.py
  Diff comments   View file @4c0f274
... ... @@ -17,7 +17,7 @@ for short_path in short_paths:
17 17  
18 18 # Project imports
19 19 from majordome.agent.Agent import Agent, build_agent
20   -from user_mgmt.models import PyrosUser, SP_Period, Period, SP_Period, SP_Period_Guest, SP_PeriodWorkflow
  20 +from user_mgmt.models import PyrosUser, Institute, SP_Period, Period, SP_Period, SP_Period_Guest, SP_PeriodWorkflow
21 21 import vendor.guitastro.src.guitastro as guitastro
22 22  
23 23 # Django imports
... ... @@ -229,6 +229,9 @@ class A_SCP_Manager(Agent):
229 229 next_sp_to_be_notified = next_sp.filter(status=SP_Period.STATUSES_ACCEPTED,is_valid = True)
230 230 self.send_mail_to_observers_for_notification(next_sp_to_be_notified)
231 231 SP_PeriodWorkflow.objects.create(period=self.period,action=SP_PeriodWorkflow.NOTIFICATION)
  232 + self.update_sun_moon_ephems()
  233 + self.set_quota_for_institutes(self.period.id)
  234 + self.set_quota_for_sp(self.period.id)
232 235  
233 236 def routine_process_body(self):
234 237 print("routine automatic period workflow")
... ... @@ -243,66 +246,27 @@ class A_SCP_Manager(Agent):
243 246 for n in range(int((end_date - start_date).days)):
244 247 yield start_date + timedelta(n)
245 248  
246   - def do_generate_ephem_moon_and_sun_for_period(self, period_id:int):
247   - # Obsolete TODO
248   - period = Period.objects.get(id=period_id)
249   - period_start_date = period.start_date
250   - period_end_date = period.end_date
251   -
252   - root_path = os.environ.get("PROJECT_ROOT_PATH")
253   - os.chdir(root_path)
254   -
255   - home = guitastro.Home(self._oc["config"].getHome())
256   - self._fn.longitude(home.longitude)
257   -
258   - ephem_data_night_folder = self._fn.rootdir
259   - if os.path.exists(ephem_data_night_folder):
260   - period_id = str(period.id)
261   - # form correct period string
262   - if len(str(period.id)) < 3:
263   - while len(period_id) < 3:
264   - period_id = "0" + period_id
265   - period_id = "P" + period_id
266   - ephem_data_night_folder = os.path.join(ephem_data_night_folder, period_id)
267   - if not os.path.exists(ephem_data_night_folder):
268   - os.makedirs(ephem_data_night_folder, exist_ok=True)
269   - for single_date in self.daterange(period_start_date, period_end_date):
270   - current_date = single_date.strftime("%Y%m%d")
271   -
272   - eph = guitastro.Ephemeris()
273   - target_sun = "sun"
274   - ephem_sun = eph.target2night(target_sun, current_date, None, None)
275   -
276   - self._fn.fcontext_create("pyros_eph", "Ephemeris PyROS")
277   - self._fn.fcontext = "pyros_eph"
278   - self._fn.pathnaming("PyROS.eph.1")
279   - self._fn.rootdir = "/tmp/eph"
280   - self._fn.extension = ".f"
281   - param = {}
282   - param['period'] = period_id
283   - param['date'] = current_date
284   - param['unit'] = self.pconfig.unit_name
285   - param['version'] = 1
286   - param['target'] = "sun"
287   - fname = self._fn.naming_set(param)
288   - file_name_sun = self._fn.join(fname)
289   -
290   - # moon parameters
291   - param['target'] = "moon"
292   - fname = self._fn.naming_set(param)
293   - file_name_moon = self._fn.join(fname)
294   - os.chdir(ephem_data_night_folder)
295   - pickle.dump(ephem_sun, open(f"{file_name_sun}.f","wb"))
296   - target_moon = "moon"
297   - ephem_moon = eph.target2night(target_moon, current_date, None, None)
298   - pickle.dump(ephem_moon, open(f"{file_name_moon}.f","wb"))
299   -
300   -
301   -
302   -
303   - # lire tous les fichiers sun de la period et appliquer le sky_elev pour déterminer le quota total de période
304   - # prendre le champ alt &
305   - # -> prendre tous les éléments en dessous de duskelev (cf l412 d'A_Sheduler, exemple 14 des éphémérides pour trier ces éléménts) et faire un sum
  249 + def set_quota_for_institutes(self, id_period):
  250 + for institute in Institute.objects.all():
  251 + quota_f = institute.quota_f
  252 + # the lowest id of quota table for this period should be the first night of the period
  253 + period_quota = Period.objects.get(id=id_period).quota
  254 + institute_quota = period_quota.convert_to_quota(quota_f)
  255 + new_quota = Quota()
  256 + new_quota.set_attributes_and_save(institute_quota)
  257 + institute.quota = new_quota
  258 + institute.save()
  259 +
  260 + def set_quota_for_SP(self, id_period):
  261 + period = Period.objects.get(id=id_period)
  262 + for sp_period in SP_Period.objects.filter(period=period)
  263 + sp = sp_period.scientific_program
  264 + institute = sp.institute
  265 + institute_quota = institute.quota
  266 + new_quota = Quota()
  267 + quota_attributes = institute_quota.convert_to_quota(sp.quota_f)
  268 + new_quota.set_attributes_and_save(quota_attributes)
  269 +
306 270  
307 271 if __name__ == "__main__":
308 272  
... ... @@ -313,7 +277,7 @@ if __name__ == &quot;__main__&quot;:
313 277 print("ARGV OF AGENT SP :",sys.argv)
314 278 if len(sys.argv) > 1 and sys.argv[1] == "test":
315 279 print("i'm in test")
316   - agentSP = AgentSP(use_db_test=True)
  280 + agentSP = A_SCP_Manager(use_db_test=True)
317 281 agentSP.run()
318 282 #agent = build_agent(agentSP, RUN_IN_THREAD=True)
319 283 else:
... ...
src/core/pyros_django/scp_mgmt/forms.py
  Diff comments   View file @4c0f274
... ... @@ -39,13 +39,13 @@ class SP_PeriodForm(forms.ModelForm):
39 39 fields = (
40 40 #"period",
41 41 "public_visibility",
42   - "quota_minimal",
43   - "quota_nominal",
44   - "quota_allocated",
45   - "over_quota_duration",
46   - "over_quota_duration_allocated",
47   - "token",
48   - "token_allocated",
  42 + # "quota_minimal",
  43 + # "quota_nominal",
  44 + # "quota_allocated",
  45 + # "over_quota_duration",
  46 + # "over_quota_duration_allocated",
  47 + # "token",
  48 + # "token_allocated",
49 49 "vote_referee1",
50 50 "reason_referee1",
51 51 "vote_referee2",
... ...
src/core/pyros_django/scp_mgmt/models.py
  Diff comments   View file @4c0f274
1   -#from django.db import models
  1 +from django.db import models
  2 +
  3 +
  4 +class Quota(models.Model):
  5 + id_period = models.BigIntegerField(blank=True, null=True)
  6 + night_id = models.CharField(max_length=8, null=True, blank=True, db_index=True)
  7 + d_totalq = models.BigIntegerField(default=0, blank=True, null=True)
  8 + d_totalx = models.BigIntegerField(default=0, blank=True, null=True)
  9 +
  10 + d_previousq = models.BigIntegerField(default=0, blank=True, null=True)
  11 + d_previousx = models.BigIntegerField(default=0, blank=True, null=True)
  12 +
  13 + d_currentq = models.BigIntegerField(default=0, blank=True, null=True)
  14 + d_currentx = models.BigIntegerField(default=0, blank=True, null=True)
  15 +
  16 + d_passedq = models.BigIntegerField(default=0, blank=True, null=True)
  17 + d_passedx = models.BigIntegerField(default=0, blank=True, null=True)
  18 +
  19 + d_scheduleq = models.BigIntegerField(default=0, blank=True, null=True)
  20 + d_schedulex = models.BigIntegerField(default=0, blank=True, null=True)
  21 +
  22 + d_nextq = models.BigIntegerField(default=0, blank=True, null=True)
  23 + d_nextx = models.BigIntegerField(default=0, blank=True, null=True)
  24 +
  25 + @property
  26 + def d_total(self):
  27 + return self.d_totalq + self.d_totalx
  28 +
  29 + @property
  30 + def d_previous(self):
  31 + return self.d_previousq + self.d_previousx
  32 +
  33 + @property
  34 + def d_current(self):
  35 + return self.d_currentq + self.d_currentx
  36 +
  37 + @property
  38 + def d_passed(self):
  39 + return self.d_passedq + self.d_passedx
  40 +
  41 + @property
  42 + def d_schedule(self):
  43 + return self.d_scheduleq + self.d_schedulex
  44 +
  45 + @property
  46 + def d_next(self):
  47 + return self.d_nextq + self.d_nextx
  48 +
  49 + def set_attributes_and_save(self, quota_attributes:dict):
  50 +
  51 + if quota_attributes.get("id_period") != None:
  52 + self.id_period = quota_attributes["id_period"]
  53 + if quota_attributes.get("night_id") != None:
  54 + self.night_id = quota_attributes["night_id"]
  55 + if quota_attributes.get("d_totalq") != None:
  56 + self.d_totalq = quota_attributes["d_totalq"]
  57 + if quota_attributes.get("d_totalx") != None:
  58 + self.d_totalx = quota_attributes["d_totalx"]
  59 + if quota_attributes.get("d_previousq") != None:
  60 + self.d_previousq = quota_attributes["d_previousq"]
  61 + if quota_attributes.get("d_previousx") != None:
  62 + self.d_previousx = quota_attributes["d_previousx"]
  63 + if quota_attributes.get("d_currentq") != None:
  64 + self.d_currentq = quota_attributes["d_currentq"]
  65 + if quota_attributes.get("d_currentx") != None:
  66 + self.d_currentx = quota_attributes["d_currentx"]
  67 + if quota_attributes.get("d_scheduleq") != None:
  68 + self.d_scheduleq = quota_attributes["d_scheduleq"]
  69 + if quota_attributes.get("d_schedulex") != None:
  70 + self.d_schedule = quota_attributes["d_schedulex"]
  71 + if quota_attributes.get("d_nextq") != None:
  72 + self.d_nextq = quota_attributes["d_nextq"]
  73 + if quota_attributes.get("d_nextx") != None:
  74 + self.d_nextx = quota_attributes["d_nextx"]
  75 + self.save()
  76 +
  77 + def convert_to_quota(self, quota_f):
  78 + quota_institute = {}
  79 +
  80 + quota_institute["d_totalq"] = self.d_totalq * quota_f
  81 + quota_institute["d_totalx"] = self.d_totalx * quota_f
  82 + quota_institute["d_previousq"] = self.d_previousq * quota_f
  83 + quota_institute["d_previousx"] = self.d_previousx * quota_f
  84 + quota_institute["d_currentq"] = self.d_currentq * quota_f
  85 + quota_institute["d_currentx"] = self.d_currentx * quota_f
  86 + quota_institute["d_passedq"] = self.d_passedq * quota_f
  87 + quota_institute["d_passedx"] = self.d_passedx * quota_f
  88 + quota_institute["d_scheduleq"] = self.d_scheduleq * quota_f
  89 + quota_institute["d_schedulex"] = self.d_schedulex * quota_f
  90 + quota_institute["d_nextq"] = self.d_nextq * quota_f
  91 + quota_institute["d_nextx"] = self.d_nextx * quota_f
  92 +
  93 + return quota_institute
... ...
src/core/pyros_django/seq_submit/models.py
  Diff comments   View file @4c0f274
... ... @@ -34,6 +34,7 @@ from django.db.models import Q
34 34  
35 35 # Project imports
36 36 from user_mgmt.models import PyrosUser, ScientificProgram, Period
  37 +from scp_mgmt.models import Quota
37 38 # DeviceCommand is used by class Command
38 39 sys.path.append("../../..")
39 40 from vendor.guitastro.src.guitastro import FileNames, Ima
... ... @@ -349,8 +350,9 @@ class Sequence(models.Model):
349 350 obsolete = models.BooleanField(default=False)
350 351 processing = models.BooleanField(default=False)
351 352 flag = models.CharField(max_length=45, blank=True, null=True)
352   - period = models.ForeignKey(Period, on_delete=models.DO_NOTHING, related_name="sequences", blank=True, null=True)
  353 + period = models.ForeignKey(Period, on_delete=models.DO_NOTHING, related_name="sequence_period", blank=True, null=True)
353 354 #period = models.ForeignKey("Period", on_delete=models.DO_NOTHING, related_name="sequences", blank=True, null=True)
  355 + quota = models.ForeignKey(Quota, on_delete=models.DO_NOTHING,related_name="sequence_quotas", blank=True, null=True)
354 356  
355 357 start_date = models.DateTimeField(
356 358 blank=True, null=True, default=timezone.now, editable=True)
... ...
src/core/pyros_django/user_mgmt/models.py
  Diff comments   View file @4c0f274
... ... @@ -37,6 +37,11 @@ from django.db.models.query import QuerySet
37 37 from django.utils import timezone
38 38  
39 39  
  40 +# Pyros import
  41 +
  42 +from scp_mgmt.models import Quota
  43 +
  44 +
40 45  
41 46  
42 47 # Project imports
... ... @@ -52,7 +57,6 @@ class UserLevel(models.Model):
52 57 name = models.CharField(max_length=45, blank=True, null=True)
53 58 desc = models.TextField(blank=True, null=True)
54 59 priority = models.IntegerField(blank=True, null=True)
55   - quota = models.FloatField(blank=True, null=True)
56 60  
57 61 class Meta:
58 62 managed = True
... ... @@ -79,9 +83,11 @@ class Country(models.Model):
79 83 class Institute(models.Model):
80 84 name = models.CharField(max_length=100, blank=False,
81 85 null=False, unique=True)
82   - # quota %
83   - quota = models.IntegerField(
84   - validators=[MinValueValidator(0), MaxValueValidator(100)])
  86 + # fraction quota
  87 + quota_f = models.FloatField(
  88 + validators=[MinValueValidator(0), MaxValueValidator(1)], blank=True, null=True)
  89 +
  90 + quota = models.ForeignKey(Quota, on_delete=models.DO_NOTHING,related_name="institute_quotas", blank=True, null=True)
85 91 #representative_user = models.ForeignKey("PyrosUser", on_delete=models.DO_NOTHING,related_name="institutes",default=1)
86 92  
87 93 def __str__(self) -> str:
... ... @@ -90,6 +96,8 @@ class Institute(models.Model):
90 96 class Meta:
91 97 db_table = "institute"
92 98  
  99 +
  100 +
93 101 class ScienceTheme(models.Model):
94 102 name = models.CharField(max_length=120, blank=False, null=False, default="", unique=True)
95 103 def __str__(self) -> str:
... ... @@ -367,6 +375,8 @@ class Period(models.Model):
367 375 data_accessibility_duration = models.PositiveIntegerField(
368 376 blank=True, null=True, default=365*10, editable=True)
369 377  
  378 + quota = models.ForeignKey(Quota, on_delete=models.DO_NOTHING,related_name="period_quotas", blank=True, null=True)
  379 +
370 380 @property
371 381 def end_date(self):
372 382 return self.start_date + relativedelta(days=self.exploitation_duration)
... ... @@ -461,10 +471,13 @@ class ScientificProgram(models.Model):
461 471 description_short = models.TextField(default="", max_length=320)
462 472 description_long = models.TextField(default="")
463 473 institute = models.ForeignKey('Institute', on_delete=models.DO_NOTHING, related_name="scientific_programs")
464   - sp_pi = models.ForeignKey('PyrosUser', on_delete=models.DO_NOTHING, related_name="Scientific_Program_Users")
  474 + sp_pi = models.ForeignKey('PyrosUser', on_delete=models.DO_NOTHING, related_name="scientific_Program_Users")
465 475 science_theme = models.ForeignKey(ScienceTheme, on_delete=models.DO_NOTHING, related_name="scientific_program_theme", default=1)
466 476 is_auto_validated = models.BooleanField(default=False)
467 477 objects = ScientificProgramManager()
  478 + quota = models.ForeignKey(Quota, on_delete=models.DO_NOTHING,related_name="scientific_program_quotas", blank=True, null=True)
  479 + quota_f = models.FloatField(
  480 + validators=[MinValueValidator(0), MaxValueValidator(1)], blank=True, null=True)
468 481  
469 482 class Meta:
470 483 managed = True
... ... @@ -525,14 +538,6 @@ class SP_Period(models.Model):
525 538 is_valid = models.TextField(
526 539 choices=IS_VALID, default=IS_VALID_REJECTED, blank=True)
527 540 status = models.TextField(choices=STATUSES, default=STATUSES_DRAFT)
528   - quota_minimal = models.PositiveIntegerField(default=0)
529   - quota_nominal = models.PositiveIntegerField(default=0)
530   - quota_allocated = models.PositiveIntegerField(default=0, blank=True)
531   - quota_remaining = models.PositiveIntegerField(default=0, blank=True)
532   - over_quota_duration = models.PositiveIntegerField(default=0)
533   - over_quota_duration_allocated = models.PositiveIntegerField(
534   - default=0, blank=True)
535   - over_quota_duration_remaining = models.PositiveIntegerField(default=0)
536 541 token = models.PositiveIntegerField(default=0)
537 542 token_allocated = models.PositiveIntegerField(default=0, blank=True)
538 543 token_remaining = models.PositiveIntegerField(default=0, blank=True)
... ...