Commit 9d8c5b5c5d2e62db1d237423c6f72d59cd884628

Authored by Etienne Pallier
1 parent 0318e3c9
Exists in dev

Toute premiere version de la generation auto de la doc sphinx (via pyros update)…

… : doc guitastro + doc codestyle
doc/codestyle_examples/codestyle_first.py
... ... @@ -14,6 +14,8 @@
14 14  
15 15 - Sphinx generalities on RST format : https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html
16 16 - Sphinx autodoc : https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html
  17 + - Sphinx autodocsumm : https://autodocsumm.readthedocs.io/en/latest/index.html
  18 + - Sphinx autosummary : https://www.sphinx-doc.org/en/master/usage/extensions/autosummary.html
17 19 - Sphinx apidoc (1) for autogeneration of files in the source folder : https://www.sphinx-doc.org/en/master/man/sphinx-apidoc.html
18 20 - Sphinx apidoc (2) more explanations : https://samnicholls.net/2016/06/15/how-to-sphinx-readthedocs
19 21 - Sphinx inheritance diagrams : https://www.sphinx-doc.org/en/master/usage/extensions/inheritance.html
... ... @@ -113,8 +115,6 @@ def general_function_that_returns_a_float(
113 115 general_function_that_returns_a_float(arg_a=1, arg_b="toto") # => OK
114 116  
115 117 general_function_that_returns_a_float(arg_a=1, arg_b=1.2) # => KO (float not allowed)
116   -
117   -
118 118 """
119 119  
120 120 # comment on a
... ...
doc/doc_rst/.gitignore 0 → 100644
... ... @@ -0,0 +1,3 @@
  1 +build*/*
  2 +TMP
  3 +source/generated_api/*
... ...
doc/doc_rst/Makefile 0 → 100644
... ... @@ -0,0 +1,20 @@
  1 +# Minimal makefile for Sphinx documentation
  2 +#
  3 +
  4 +# You can set these variables from the command line, and also
  5 +# from the environment for the first two.
  6 +SPHINXOPTS ?=
  7 +SPHINXBUILD ?= sphinx-build
  8 +SOURCEDIR = source
  9 +BUILDDIR = build
  10 +
  11 +# Put it first so that "make" without argument is like "make help".
  12 +help:
  13 + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
  14 +
  15 +.PHONY: help Makefile
  16 +
  17 +# Catch-all target: route all unknown targets to Sphinx using the new
  18 +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
  19 +%: Makefile
  20 + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
... ...
doc/doc_rst/make.bat 0 → 100644
... ... @@ -0,0 +1,35 @@
  1 +@ECHO OFF
  2 +
  3 +pushd %~dp0
  4 +
  5 +REM Command file for Sphinx documentation
  6 +
  7 +if "%SPHINXBUILD%" == "" (
  8 + set SPHINXBUILD=sphinx-build
  9 +)
  10 +set SOURCEDIR=source
  11 +set BUILDDIR=build
  12 +
  13 +if "%1" == "" goto help
  14 +
  15 +%SPHINXBUILD% >NUL 2>NUL
  16 +if errorlevel 9009 (
  17 + echo.
  18 + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
  19 + echo.installed, then set the SPHINXBUILD environment variable to point
  20 + echo.to the full path of the 'sphinx-build' executable. Alternatively you
  21 + echo.may add the Sphinx directory to PATH.
  22 + echo.
  23 + echo.If you don't have Sphinx installed, grab it from
  24 + echo.http://sphinx-doc.org/
  25 + exit /b 1
  26 +)
  27 +
  28 +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
  29 +goto end
  30 +
  31 +:help
  32 +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
  33 +
  34 +:end
  35 +popd
... ...
doc/doc_rst/make_rst_then_html 0 → 100755
... ... @@ -0,0 +1,44 @@
  1 +
  2 +# A - Generate RST files
  3 +# ---------------------------------
  4 +
  5 +#SPHINX_APIDOC_OPTIONS=autosummary,autosummary-no-nesting,members,undoc-members,show-inheritance
  6 +##export SPHINX_APIDOC_OPTIONS=members,undoc-members,show-inheritance,autosummary,autosummary-no-nesting
  7 +
  8 +#sphinx-apidoc -f -o source ../../src/core
  9 +
  10 +# 1) guitastro package (celme)
  11 +# Doc au format de Alain
  12 +cd ../../vendor/guitastro/doc_rst/
  13 +pwd
  14 +make html
  15 +# Doc au format de Etienne (just uncomment 1st line)
  16 +###sphinx-apidoc -f -o source/generated_api -a --separate --tocfile guitastro_package ../../vendor/guitastro
  17 +###sphinx-apidoc -f -o source/generated_api -a --separate --module-first --tocfile guitastro_package ../../vendor/guitastro
  18 +#sphinx-apidoc -f -o source -a --separate --module-first --tocfile guitastro_package ../../vendor/guitastro
  19 +#sphinx-apidoc -f -o source -a --separate --module-first --tocfile celme_package ../../src/core/celme
  20 +#sphinx-apidoc -o source --tocfile celme_package ../../src/core/celme
  21 +cd -
  22 +pwd
  23 +
  24 +# 2) device_controller package
  25 +######sphinx-apidoc -o source -a --separate --module-first --tocfile device_controller_package ../../src/device_controller
  26 +
  27 +# 3) pyros_django package
  28 +####sphinx-apidoc -f -o source -a --separate --module-first --tocfile pyros_django_package ../../src/core/pyros_django
  29 +#sphinx-apidoc -f -o source --tocfile pyros_django_package ../../src/core/pyros_django
  30 +
  31 +# 4) Code style package
  32 +sphinx-apidoc -f -o source/generated_api -a --separate --module-first --tocfile codestyle_examples_package ../codestyle_examples
  33 +##sphinx-apidoc -f -o source -a --separate --module-first --tocfile codestyle_examples_package ../codestyle_examples
  34 +#sphinx-apidoc -o source -a --separate --module-first --tocfile code_style_package ../code_style/my_package1
  35 +
  36 +#sphinx-apidoc -f -o source ../../src
  37 +#sphinx-apidoc -f -o --implicit-namespaces source ../../src/
  38 +
  39 +
  40 +# B - Generate HTML (and pdf) files (from RST)
  41 +# ---------------------------------
  42 +
  43 +sphinx-build -b html source build/html/
  44 +#sphinx-build -b pdf source build
... ...
doc/doc_rst/requirements.txt 0 → 100644
... ... @@ -0,0 +1,15 @@
  1 +Sphinx
  2 +sphinx-autodoc-typehints
  3 +sphinx-pyreverse
  4 +
  5 +# Extensions
  6 +autodocsumm
  7 +
  8 +# Themes
  9 +#alabaster
  10 +sphinx-rtd-theme
  11 +
  12 +# Builder
  13 +rst2pdf
  14 +#rstcheck
  15 +
... ...
doc/doc_rst/source/conf.py 0 → 100644
... ... @@ -0,0 +1,214 @@
  1 +# Configuration file for the Sphinx documentation builder.
  2 +#
  3 +# This file only contains a selection of the most common options. For a full
  4 +# list see the documentation:
  5 +# https://www.sphinx-doc.org/en/master/usage/configuration.html
  6 +
  7 +# -- Path setup --------------------------------------------------------------
  8 +
  9 +# If extensions (or modules to document with autodoc) are in another directory,
  10 +# add these directories to sys.path here. If the directory is relative to the
  11 +# documentation root, use os.path.abspath to make it absolute, like shown here.
  12 +#
  13 +import os
  14 +import sys
  15 +# sys.path.insert(0, os.path.abspath('.'))
  16 +
  17 +#import pathlib
  18 +#sys.path.insert(0, pathlib.Path(__file__).parents[2].resolve().as_posix())
  19 +
  20 +# - for code style examples :
  21 +sys.path.insert(0, os.path.abspath('../../../doc'))
  22 +# - for PyROS :
  23 +sys.path.insert(0, os.path.abspath('../../..'))
  24 +sys.path.insert(0, os.path.abspath('../../../src'))
  25 +sys.path.insert(0, os.path.abspath('../../../src/core'))
  26 +# - for guitastro :
  27 +sys.path.insert(0, os.path.abspath('../../../vendor'))
  28 +#sys.path.insert(0, os.path.abspath('../../../src/device_controller'))
  29 +#sys.path.insert(0, os.path.abspath('../../../src/core/pyros_django'))
  30 +#sys.path.insert(0, os.path.abspath('../../../src/core/celme'))
  31 +sys.setrecursionlimit(1500)
  32 +
  33 +
  34 +# -- Project information -----------------------------------------------------
  35 +
  36 +project = 'PyROS'
  37 +copyright = '2022, E. Pallier, A. Klotz, A. Koralewski'
  38 +author = 'E. Pallier, A. Klotz, A. Koralewski'
  39 +
  40 +# The full version, including alpha/beta/rc tags
  41 +release = '0.4.1'
  42 +
  43 +
  44 +# -- General configuration ---------------------------------------------------
  45 +
  46 +# Add any Sphinx extension module names here, as strings. They can be
  47 +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
  48 +# ones.
  49 +
  50 +extensions = [
  51 + 'sphinx.ext.autodoc', # automatically generate documentation for modules
  52 +
  53 + # (EP) sphinx-autodoc-typehints
  54 + # https://pypi.org/project/sphinx-autodoc-typehints
  55 + # TODO: pip install sphinx-autodoc-typehints
  56 + # TODO: make sure you load sphinx.ext.napoleon first, before sphinx-autodoc-typehints
  57 + 'sphinx.ext.napoleon', # to read Google-style or Numpy-style docstrings
  58 + 'sphinx_autodoc_typehints',
  59 +
  60 + 'sphinx.ext.todo',
  61 +
  62 + #'sphinx.ext.duration',
  63 + 'sphinx.ext.doctest',
  64 +
  65 + #'sphinx.ext.autosummary',
  66 + "autodocsumm", # to generate tables of functions, attributes, methods, etc. (python3 -m pip install autodocsumm)
  67 + 'sphinx.ext.intersphinx',
  68 + 'sphinx.ext.mathjax',
  69 + 'sphinx.ext.viewcode', # to allow vieing the source code in the web page
  70 + 'sphinx.ext.graphviz',
  71 + #'sphinx_pyreverse',
  72 + 'rst2pdf.pdfbuilder',
  73 + # to be used with option : show-inheritance-diagram (mais ne marche pas pour l'instant...)
  74 + 'sphinx.ext.inheritance_diagram',
  75 +]
  76 +
  77 +'''
  78 +try:
  79 + extensions.append('sphinx_pyreverse')
  80 +except ModuleNotFoundError:
  81 + pass # pip install sphinx_pyreverse
  82 +
  83 +try:
  84 + import rst2pdf
  85 + extensions.append('rst2pdf.pdfbuilder')
  86 +except ModuleNotFoundError:
  87 + pass # no rst2pdf for you
  88 +'''
  89 +
  90 +# (EP) To avoid sorting of methods & attributes, very annoying... (so, keep them in samed order as in source)
  91 +#autodoc_member_order = 'bysource'
  92 +
  93 +# (EP) Default config for autodoc
  94 +#'members': 'var1, var2',
  95 +#'special-members': '__init__, __str__',
  96 +#'special-members': '__str__',
  97 +#'autoclass_content': 'class',
  98 +#'exclude-members': '__init__, __weakref__',
  99 +autodoc_default_options = {
  100 +
  101 + # make autodocsumm active by default for all autodoc directives
  102 + 'autosummary': True,
  103 +
  104 + # autodoc will also generate document for the undoc-umented members (not having docstrings)
  105 + 'undoc-members': True,
  106 +
  107 + 'member-order': 'bysource',
  108 + 'special-members': '__str__',
  109 + #'special-members': '__init__, __str__',
  110 + #'exclude-members': '__weakref__',
  111 + 'show-inheritance': True,
  112 + #'show-inheritance': False,
  113 + # ne marche pas ?
  114 + 'show-inheritance-diagram': True,
  115 +
  116 + 'typehints_fully_qualified': False,
  117 +
  118 + # (EP) nothing below is taken into account...
  119 + ##'add_module_names': False,
  120 + ##'autoclass_content': 'init',
  121 + #'autoclass_content': 'both',
  122 + #'autodoc_typehints': 'signature',
  123 + ##'autodoc_typehints': 'both',
  124 + #'autodoc_typehints': 'description',
  125 + ##'autodoc_typehints_description_target': 'all',
  126 + #'autodoc_class_signature': 'mixed',
  127 + #'autodoc_class_signature': 'separated',
  128 +}
  129 +#print(autodoc_default_options)
  130 +#exit
  131 +
  132 +# (EP) necessary because not taken into account in autodoc_default_options above !!!!!!
  133 +
  134 +
  135 +# - short module name
  136 +add_module_names = False
  137 +
  138 +# Le commentaire de la methode __init__ est rajouté au commentaire de la classe ?
  139 +# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#confval-autoclass_content
  140 +# - NON
  141 +#autoclass_content = 'class'
  142 +# - OUI
  143 +#autoclass_content = 'init'
  144 +# - OUI, les 2 commentaires (class et init) sont fusionnés
  145 +autoclass_content = 'both'
  146 +
  147 +# - Documenter automatiquement TOUS les arguments de methodes, meme ceux qui ne sont pas explicitement documentés
  148 +#autodoc_typehints = 'both'
  149 +#autodoc_typehints = 'signature'
  150 +# Show types only in descriptions, not in signatures
  151 +autodoc_typehints = "description"
  152 +
  153 +# don't include docstrings from the parent class
  154 +autodoc_inherit_docstrings = False
  155 +
  156 +# This value controls whether the types of undocumented parameters and return values are documented when autodoc_typehints is set to description.
  157 +# The default value is "all", meaning that types are documented for all parameters and return values, whether they are documented or not.
  158 +# When set to "documented", types will only be documented for a parameter or a return value that is already documented by the docstring.
  159 +##autodoc_typehints_description_target = 'all'
  160 +
  161 +# Afficher le nom de la classe AVEC ses arguments ou SANS ?
  162 +# - AVEC
  163 +autodoc_class_signature = 'mixed'
  164 +# - SANS
  165 +#autodoc_class_signature = 'separated'
  166 +
  167 +
  168 +# Napoleon settings
  169 +# https://www.sphinx-doc.org/en/master/usage/extensions/napoleon.html#configuration
  170 +napoleon_google_docstring = True
  171 +napoleon_numpy_docstring = False
  172 +#napoleon_include_init_with_doc = False
  173 +'''
  174 +napoleon_include_private_with_doc = False
  175 +napoleon_include_special_with_doc = True
  176 +napoleon_use_admonition_for_examples = False
  177 +napoleon_use_admonition_for_notes = False
  178 +napoleon_use_admonition_for_references = False
  179 +napoleon_use_ivar = False
  180 +napoleon_use_param = True
  181 +napoleon_use_rtype = True
  182 +napoleon_preprocess_types = False
  183 +napoleon_type_aliases = None
  184 +napoleon_attr_annotations = True
  185 +'''
  186 +
  187 +# Add any paths that contain templates here, relative to this directory.
  188 +templates_path = ['_templates']
  189 +
  190 +# List of patterns, relative to source directory, that match files and
  191 +# directories to ignore when looking for source files.
  192 +# This pattern also affects html_static_path and html_extra_path.
  193 +#exclude_patterns = []
  194 +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
  195 +
  196 +
  197 +# -- Options for HTML output -------------------------------------------------
  198 +
  199 +# The theme to use for HTML and HTML Help pages. See the documentation for
  200 +# a list of builtin themes.
  201 +#
  202 +#html_theme = 'alabaster'
  203 +# (EP) Read The Docs theme
  204 +html_theme = 'sphinx_rtd_theme'
  205 +# Set html_theme to default for “standard” ReadTheDocs theme
  206 +#html_theme = 'default'
  207 +#html_theme = 'nature'
  208 +
  209 +# Add any paths that contain custom static files (such as style sheets) here,
  210 +# relative to this directory. They are copied after the builtin static files,
  211 +# so a file named "default.css" will overwrite the builtin "default.css".
  212 +html_static_path = ['_static']
  213 +
  214 +autosummary_generate = True
... ...
doc/doc_rst/source/index.rst 0 → 100644
... ... @@ -0,0 +1,97 @@
  1 +*****************************************************
  2 +Welcome in the PyROS developer documentation
  3 +*****************************************************
  4 +
  5 +.. image:: ./doc_images/logo-irap-couleur-1024x386.png
  6 + :height: 100px
  7 + :align: center
  8 +
  9 ++------------------------------+------------------------------------------------------------------+
  10 +| | IRAP | | Project Scientist (PI) : Alain Klotz - aklotz AT irap.omp.eu |
  11 +| | 9 Avenue Colonel Roche | | Project Manager : Etienne Pallier - epallier AT irap.omp.eu |
  12 +| | 31028 Toulouse Cedex 4 | | Developper : Alexis Koralewski - akoralewski AT irap.omp.eu |
  13 +| | FRANCE | | http://pyros.irap.omp.eu |
  14 ++------------------------------+------------------------------------------------------------------+
  15 +
  16 +.. warning:: This is a warning message, if needed
  17 +
  18 +
  19 +
  20 +
  21 +
  22 +1. PyROS files
  23 +**************
  24 +
  25 +See in the Installation documentation - Project structure <https://docs.google.com/document/d/1DjT_rAzeVPl9SnQG3_20GFbDf67JEUAsOD1uhDMlOVw/edit#heading=h.19nwnkgoi5k>`_
  26 +
  27 +.. comments
  28 + .. toctree::
  29 + :maxdepth: 3
  30 +
  31 + using_mount_classes
  32 +
  33 +2. PyROS configuration
  34 +**********************
  35 +
  36 +See in the Design&Implementation documentation - PyROS configuration <https://docs.google.com/document/d/1osHT3d8GiofRd9k3nL7cz4Hcmo9WVEOcSaSSQpgJcFs/edit#heading=h.21n0fso0r8u9>`_
  37 +
  38 +
  39 +
  40 +3. Observatory configuration
  41 +****************************
  42 +
  43 +See in the Design&Implementation documentation - Observatory configuration <https://docs.google.com/document/d/1osHT3d8GiofRd9k3nL7cz4Hcmo9WVEOcSaSSQpgJcFs/edit#heading=h.ps1odi23szui>`_
  44 +
  45 +
  46 +
  47 +4. PyROS API Source Code documentation
  48 +**************************************
  49 +
  50 +The PyROS Packages, Classes, and Methods
  51 +
  52 +
  53 +
  54 +5. Guitastro doc
  55 +****************
  56 +
  57 +`Guitastro documentation <../../../../vendor/guitastro/doc_rst/build/html/index.html>`_
  58 +
  59 +
  60 +
  61 +
  62 +
  63 +
  64 +.. COMMENT: The toctree directive is mainly to let Sphinx be aware of the document hierarchy (the sidebar will reflect this understanding)
  65 +
  66 +
  67 +.. toctree::
  68 + :maxdepth: 3
  69 + :caption: Contents:
  70 +
  71 + generated_api/codestyle_examples_package
  72 +
  73 +.. comment guitastro
  74 +
  75 +.. comment `Guitastro documentation <../../../vendor/guitastro/doc_rst/build/html/index.html>`_
  76 +
  77 +.. comment ../../../vendor/guitastro/doc_rst/source/index
  78 +
  79 +.. Generation doc guitastro API directement depuis ce script (EP) :
  80 +.. generated_api/guitastro_package
  81 +
  82 +.. Autres modules :
  83 +.. device_controller_package
  84 +.. pyros_django_package
  85 +.. autodoc_pyros
  86 +.. pyros-doc-DEV
  87 +.. modules
  88 +
  89 +
  90 +Indices and tables
  91 +==================
  92 +
  93 +* :ref:`genindex`
  94 +* :ref:`modindex`
  95 +
  96 +.. * :ref:`search`
  97 +
... ...
pyros.py
... ... @@ -9,24 +9,11 @@
9 9  
10 10 ''' *** STANDARD IMPORTS *** '''
11 11  
12   -import argparse
13   -import fileinput
14   -import glob
15   -import os
16   -import platform
17   -import shutil
18   -import signal
19   -import subprocess
20   -import sys
21   -import time
22   -import re
23   -import glob
24 12 #import logging
25 13  
26 14  
27 15 ''' *** PROJECT IMPORTS *** '''
28 16 #from src.pyros_logger import *
29   -from src.pyros_logger import log
30 17 #from src.pyros_logger import log, log_d,log_i,log_w,log_e,log_c
31 18 #from src import pyros_logger
32 19  
... ... @@ -39,6 +26,19 @@ from src.pyros_logger import log
39 26  
40 27 #DEBUG = False
41 28  
  29 +import argparse
  30 +import fileinput
  31 +import glob
  32 +import os
  33 +import platform
  34 +import shutil
  35 +import signal
  36 +import subprocess
  37 +import sys
  38 +import time
  39 +import re
  40 +import glob
  41 +from src.pyros_logger import log
42 42 UPDATE = False
43 43  
44 44 PYROS_DJANGO_BASE_DIR = "src/core/pyros_django"
... ... @@ -47,26 +47,26 @@ INIT_FIXTURE = &quot;initial_fixture_TZ.json&quot;
47 47 _previous_dir = None
48 48  
49 49 AGENTS = {
50   - #"agentX" : "agent",
51   - "agent" : "Agent",
52   - "agent2" : "Agent",
53   - "agentX" : "AgentX",
54   - "agentA" : "AgentA",
55   - "agentB" : "AgentB",
56   - "agentC" : "AgentC",
57   - "agentM" : "AgentM",
58   - "agentSP" : "AgentSP",
59   - #"agentDevice" : "AgentDevice",
60   - #"agentDeviceTelescopeGemini" : "AgentDeviceTelescopeGemini",
61   - "agentDeviceGemini" : "AgentDeviceGemini",
62   - "agentDeviceSBIG" : "AgentDeviceSBIG",
63   - "agentTelescopeRequester" : "AgentTelescopeRequester",
64   - "agentMultiRequester" : "AgentMultiRequester",
65   - "webserver" : "webserver",
66   - "monitoring" : "monitoring",
67   - "majordome" : "majordome",
68   - "scheduler" : "scheduler",
69   - "alert_manager" : "alert_manager"
  50 + # "agentX" : "agent",
  51 + "agent": "Agent",
  52 + "agent2": "Agent",
  53 + "agentX": "AgentX",
  54 + "agentA": "AgentA",
  55 + "agentB": "AgentB",
  56 + "agentC": "AgentC",
  57 + "agentM": "AgentM",
  58 + "agentSP": "AgentSP",
  59 + # "agentDevice" : "AgentDevice",
  60 + # "agentDeviceTelescopeGemini" : "AgentDeviceTelescopeGemini",
  61 + "agentDeviceGemini": "AgentDeviceGemini",
  62 + "agentDeviceSBIG": "AgentDeviceSBIG",
  63 + "agentTelescopeRequester": "AgentTelescopeRequester",
  64 + "agentMultiRequester": "AgentMultiRequester",
  65 + "webserver": "webserver",
  66 + "monitoring": "monitoring",
  67 + "majordome": "majordome",
  68 + "scheduler": "scheduler",
  69 + "alert_manager": "alert_manager"
70 70 }
71 71 #AGENTS = ["agentX", "webserver", "monitoring", "majordome", "scheduler", "alert_manager"]
72 72 #AGENTS = ["all", "webserver", "monitoring", "majordome", "scheduler", "alert"]
... ... @@ -86,23 +86,23 @@ if IS_WINDOWS:
86 86 PYTHON = "python.exe"
87 87  
88 88  
89   -try :
  89 +try:
90 90 # WITH_DOCKER is an environment varialbe from our Docker image
91   - WITH_DOCKER=os.environ['WITH_DOCKER']
  91 + WITH_DOCKER = os.environ['WITH_DOCKER']
92 92 except KeyError:
93   - WITH_DOCKER=False
  93 + WITH_DOCKER = False
94 94  
95   -if type(WITH_DOCKER) is str and re.match("^y$|^Y$|^yes$|^Yes$",WITH_DOCKER.rstrip()) != None :
96   - WITH_DOCKER = True
97   -else :
  95 +if type(WITH_DOCKER) is str and re.match("^y$|^Y$|^yes$|^Yes$", WITH_DOCKER.rstrip()) != None:
  96 + WITH_DOCKER = True
  97 +else:
98 98 WITH_DOCKER = False
99 99  
100 100 my_abs_path = os.path.dirname(os.path.realpath(__file__))
101 101 #VENV_ROOT = "private"
102   -if WITH_DOCKER :
  102 +if WITH_DOCKER:
103 103 VENV_ROOT = ""
104 104 VENV = ""
105   - VENV_BIN = ""
  105 + VENV_BIN = ""
106 106 else:
107 107 VENV_ROOT = "venv"
108 108 VENV = "venv_py3_pyros"
... ... @@ -116,6 +116,7 @@ else:
116 116 VENV_PYTHON = VENV_BIN + PYTHON
117 117 VENV_PIP = VENV_BIN + 'pip'
118 118  
  119 +
119 120 class Colors:
120 121 HEADER = "\033[95m"
121 122 BLUE = "\033[94m"
... ... @@ -134,9 +135,6 @@ class Colors:
134 135 LOG_BLUE = ''
135 136  
136 137  
137   -
138   -
139   -
140 138 '''
141 139 VARIABLES FOR INSTALL
142 140 '''
... ... @@ -148,7 +146,7 @@ INSTALL_DB = True
148 146 SQL_DATABASE = "pyros"
149 147 # Database automatically created by Django for tests (and automatically deleted after tests)
150 148 SQL_DATABASE_TEST = "test_pyros"
151   -# Specific database that we need(ed?) for some simulations
  149 +# Specific database that we need(ed?) for some simulations
152 150 SQL_DATABASE_SIMU = "pyros_test"
153 151 SQL_USER = ""
154 152 SQL_PSWD = ""
... ... @@ -158,8 +156,6 @@ ENV_PATH = &quot;docker/variables.env&quot;
158 156 ENV_SAMPLE_PATH = "docker/.env-sample"
159 157  
160 158  
161   -
162   -
163 159 END_OF_LINE = '\n\n'
164 160 ##VENV_BIN = '/bin/'
165 161 # --------------------------------------------
... ... @@ -171,7 +167,7 @@ if (platform.system() == &quot;Windows&quot;):
171 167 #MYSQL_EXE_PATH = "C:/Program Files (x86)/MySQL/MySQL Server 5.0/bin/"
172 168 #question = "Enter the path of the MySQL server if it is not the following name (" + MYSQL_EXE_PATH + "): "
173 169 #res = input(question)
174   - #if res!="":
  170 + # if res!="":
175 171 #MYSQL_EXE_PATH = res
176 172  
177 173 #VENV_PIP = VENV + VENV_BIN+'pip'
... ... @@ -179,16 +175,11 @@ if (platform.system() == &quot;Windows&quot;):
179 175  
180 176 # GLOBAL_PYTHON = 'python3'
181 177 GLOBAL_PYTHON = os.path.split(sys.executable)[-1]
182   -log.debug(Colors.LOG_BLUE + "Python executable is " + GLOBAL_PYTHON + Colors.END)
  178 +log.debug(Colors.LOG_BLUE + "Python executable is " +
  179 + GLOBAL_PYTHON + Colors.END)
183 180 ##if platform.dist()[0] == "centos": print("centos platform")
184 181  
185 182  
186   -
187   -
188   -
189   -
190   -
191   -
192 183 """
193 184 *************************************************************************************************
194 185 ******************** click package install (if not yet done) and import it **********************
... ... @@ -197,7 +188,7 @@ log.debug(Colors.LOG_BLUE + &quot;Python executable is &quot; + GLOBAL_PYTHON + Colors.END
197 188 # First thing first, install the click package !!!
198 189 #import fire
199 190 try:
200   - import click # https://click.palletsprojects.com
  191 + import click # https://click.palletsprojects.com
201 192 except:
202 193 #pip = "pip" if platform.system() == "Windows" else "pip3"
203 194 # TODO: "python -m pip" au lieu de "pip"
... ... @@ -211,10 +202,6 @@ except:
211 202 else:
212 203 log.error("click package installation failed")
213 204 # self.addError(self.current_command, command)
214   -
215   -
216   -
217   -
218 205  
219 206  
220 207 """
... ... @@ -223,6 +210,7 @@ except:
223 210 **************************************************************************
224 211 """
225 212  
  213 +
226 214 def _in_dir(dirname: str = ""):
227 215 return os.path.basename(os.getcwd()) == dirname
228 216  
... ... @@ -232,22 +220,23 @@ def _in_abs_dir(dirname: str = &quot;&quot;):
232 220  
233 221  
234 222 def die(msg: str = ""):
235   - log.error()
  223 + log.error('')
236 224 log.error("...ERROR...")
237 225 log.error(msg)
238   - log.error()
  226 + log.error('')
239 227 exit(1)
240   -
241 228  
242   -#TODO: implement is_async
  229 +
  230 +# TODO: implement is_async
243 231 def execProcess(command, from_venv=False, is_async=False):
244 232 from_venv_str = " from venv ("+VENV_PYTHON+")" if from_venv else ""
245 233 #printFullTerm(Colors.BLUE, "Executing command" + " [" + command + "]" + from_venv_str)
246 234 log.debug("Executing command" + " [" + command + "]" + from_venv_str)
247   - if from_venv: command = VENV_PYTHON+' ' + command
  235 + if from_venv:
  236 + command = VENV_PYTHON+' ' + command
248 237 process = subprocess.Popen(command, shell=True)
249 238 if is_async:
250   - #current_processes.append(process)
  239 + # current_processes.append(process)
251 240 return process
252 241 # Not is_async => Wait for end of execution
253 242 process.wait()
... ... @@ -257,15 +246,18 @@ def execProcess(command, from_venv=False, is_async=False):
257 246 else:
258 247 printFullTerm(Colors.WARNING, "Process execution failed")
259 248 # self.addError(self.current_command, command)
260   - #return process.returncode
261   - return True if process.returncode==0 else False
  249 + # return process.returncode
  250 + return True if process.returncode == 0 else False
  251 +
262 252  
263   -def execProcessFromVenv(command:str, is_async=False):
264   - #return execProcess(command, from_venv=True, is_async)
  253 +def execProcessFromVenv(command: str, is_async=False):
  254 + # return execProcess(command, from_venv=True, is_async)
265 255 return execProcess(command, True, is_async)
266 256  
267   -#TODO: fusionner dans execProcess avec param is_async
268   -def execProcessFromVenvAsync(command:str):
  257 +# TODO: fusionner dans execProcess avec param is_async
  258 +
  259 +
  260 +def execProcessFromVenvAsync(command: str):
269 261 return execProcessFromVenv(command, True)
270 262 """
271 263 args = command.split()
... ... @@ -280,13 +272,14 @@ def execProcessFromVenvAsync(command:str):
280 272 return p
281 273 """
282 274  
  275 +
283 276 def printColor(color: Colors, message, file=sys.stdout, eol=os.linesep, forced=False):
284 277 #system = platform.system()
285 278 """
286 279 if (self.disp == False and forced == False):
287 280 return 0
288 281 """
289   - #if system == "Windows":
  282 + # if system == "Windows":
290 283 if IS_WINDOWS:
291 284 print(message, file=file, end=eol)
292 285 else:
... ... @@ -306,7 +299,8 @@ def printFullTerm(color: Colors, string: str):
306 299 printColor(color, "-" * (columns - value))
307 300 return 0
308 301  
309   -def set_environment_variables_if_not_configured(env_path:str, env_sample_path:str)->None:
  302 +
  303 +def set_environment_variables_if_not_configured(env_path: str, env_sample_path: str) -> None:
310 304 """
311 305 Set environment variables if they aren't defined in the current environment.
312 306 Get the variables from .env file if it exists or create this file using a copy of the env_sample
... ... @@ -314,7 +308,7 @@ def set_environment_variables_if_not_configured(env_path:str, env_sample_path:st
314 308 Args:
315 309 env_path (str): path to .env file
316 310 env_sample_path (str): path to .env-sample file
317   - """
  311 + """
318 312 """
319 313 variables_names = ["MYSQL_ROOT_PASSWORD","MYSQL_ROOT_LOGIN","MYSQL_TCP_PORT","MYSQL_PYROS_LOGIN","MYSQL_PYROS_PWD","PATH_TO_OBSCONF_FILE"]
320 314 is_environment_variables_defined = True
... ... @@ -328,7 +322,7 @@ def set_environment_variables_if_not_configured(env_path:str, env_sample_path:st
328 322 """
329 323 log.debug("Start Setting environment variables")
330 324 try:
331   - with open(env_path,"r") as env_file:
  325 + with open(env_path, "r") as env_file:
332 326 # env file is empty
333 327 if os.stat(env_path).st_size == 0:
334 328 raise BaseException()
... ... @@ -338,41 +332,43 @@ def set_environment_variables_if_not_configured(env_path:str, env_sample_path:st
338 332 continue
339 333 else:
340 334 line = line.rstrip()
341   - key,value = line.split("=")
  335 + key, value = line.split("=")
342 336 # setting variables as environment variables
343 337 if WITH_DOCKER and os.environ.get(key) != value:
344   - log.warning(f"WARNING: Environment value for '{key}' needs to be uptaded (To remove this message : restart docker container to update them).'")
  338 + log.warning(
  339 + f"WARNING: Environment value for '{key}' needs to be uptaded (To remove this message : restart docker container to update them).'")
345 340 else:
346 341 # if not WITH_DOCKER and os.environ.get(key) is None:
347   - # log.error(f"ERROR: Error while reading value for '{key}'.Please fix the '{env_file}' file. For this run, PyROS will take the values from '{env_path}'")
  342 + # log.error(f"ERROR: Error while reading value for '{key}'.Please fix the '{env_file}' file. For this run, PyROS will take the values from '{env_path}'")
348 343 # raise BaseException()
349 344 if WITH_DOCKER and os.environ.get(key) is None:
350   - log.warning(f"WARNING: Environment value for '{key}' isn't defined within the container (To remove this message : restart docker container to update environment values). PyROS will take the values from '{env_path}'")
  345 + log.warning(
  346 + f"WARNING: Environment value for '{key}' isn't defined within the container (To remove this message : restart docker container to update environment values). PyROS will take the values from '{env_path}'")
351 347 raise BaseException()
352 348 os.environ[key] = value
353 349  
354 350 except:
355 351 #print(f".env not found at {env_path} or is empty, creating a file at this path from the .env-sample file stored at {ENV_SAMPLE_PATH}\nvalues from .env-sample will be used as environment variables")
356   - with open(env_sample_path,'r') as env_sample_file:
357   - with open(env_path,"w") as env_file:
  352 + with open(env_sample_path, 'r') as env_sample_file:
  353 + with open(env_path, "w") as env_file:
358 354 for env_sample_line in env_sample_file:
359 355 if(env_sample_line.startswith("#") or not env_sample_line.strip()):
360 356 continue
361   - key,value = env_sample_line.split("=")
  357 + key, value = env_sample_line.split("=")
362 358 os.environ[key] = value
363 359 env_file.write(env_sample_line)
364 360 # PYROS_DJANGO_BASE_DIR isn't an absolute path so we need to add the path to current folder to it
365   - os.environ["DJANGO_PATH"] = os.path.join(os.getcwd(),PYROS_DJANGO_BASE_DIR)
  361 + os.environ["DJANGO_PATH"] = os.path.join(
  362 + os.getcwd(), PYROS_DJANGO_BASE_DIR)
366 363 log.debug("End Setting environment variables")
367 364  
  365 +
368 366 """
369 367 else:
370 368 print("The environment variables are already configured, skipping this step...")
371 369 """
372 370  
373 371  
374   -
375   -
376 372 """
377 373 ********************************************************************************
378 374 ******************** CLI COMMANDS DEFINITION (click format) ********************
... ... @@ -404,21 +400,22 @@ def test_mode(): return GLOBAL_OPTIONS[&quot;test&quot;]
404 400 def sim_mode(): return GLOBAL_OPTIONS["sim"]
405 401 def debug_mode(): return GLOBAL_OPTIONS["debug"]
406 402  
  403 +
407 404 @click.group()
408 405 @click.option('--debug', '-d', is_flag=True, help='DEBUG mode')
409 406 @click.option('--sim', '-s', is_flag=True, help="only for an AgentDevice, asking it to start its simulator and work with it (instead of real device)")
410 407 @click.option('--test', '-t', is_flag=True, help="don't do it for real, just show what it would do")
411 408 @click.option('--verbose', '-v', is_flag=True, help='Verbose output')
412   -#@click.option('--verbose', '-v', 'verbosity', flag_value=2, default=1, help='Verbose output'),
413   -#@click.option('--quiet', '-q', 'verbosity', flag_value=0, help='Minimal output'),
414   -#@click.option('--fail-fast', '--failfast', '-f', 'fail_fast', is_flag=True, default=False, help='Stop on failure'),
  409 +# @click.option('--verbose', '-v', 'verbosity', flag_value=2, default=1, help='Verbose output'),
  410 +# @click.option('--quiet', '-q', 'verbosity', flag_value=0, help='Minimal output'),
  411 +# @click.option('--fail-fast', '--failfast', '-f', 'fail_fast', is_flag=True, default=False, help='Stop on failure'),
415 412 def pyros_launcher(debug, sim, test, verbose):
416 413 ##global log
417 414 #log.info("in pyros launcher")
418   - #pass
  415 + # pass
419 416 os.environ['PYROS_DEBUG'] = '1' if debug else '0'
420   -
421   - ##pyros_logger.set_logger_config()
  417 +
  418 + # pyros_logger.set_logger_config()
422 419 ##log = logging.getLogger('pyroslogger')
423 420 log.debug('starting pyros')
424 421 #log.info('info toto msg from pyros')
... ... @@ -426,21 +423,22 @@ def pyros_launcher(debug, sim, test, verbose):
426 423 #log.error('error toto msg from pyros')
427 424 #log.critical('error toto msg from pyros')
428 425  
429   - if debug: click.echo('DEBUG mode')
430   - if sim: click.echo('Starting and connecting to simulator instead of real device')
431   - if test: click.echo('Test mode')
432   - if verbose: click.echo('Verbose mode')
  426 + if debug:
  427 + click.echo('DEBUG mode')
  428 + if sim:
  429 + click.echo('Starting and connecting to simulator instead of real device')
  430 + if test:
  431 + click.echo('Test mode')
  432 + if verbose:
  433 + click.echo('Verbose mode')
433 434 GLOBAL_OPTIONS["debug"] = debug
434 435 GLOBAL_OPTIONS["sim"] = sim
435 436 GLOBAL_OPTIONS["test"] = test
436 437 GLOBAL_OPTIONS["verbose"] = verbose
437 438  
438 439  
439   -
440   -
441   -
442 440 @pyros_launcher.command(help="Run a pyros shell (django included)")
443   -#@global_test_options
  441 +# @global_test_options
444 442 def shell():
445 443 print("Execution commande shell")
446 444 print("NAME IS", __name__)
... ... @@ -473,9 +471,10 @@ def shell():
473 471 change_dir(PYROS_DJANGO_BASE_DIR)
474 472  
475 473 # execProcess("python install.py install")
476   - if not test_mode(): execProcessFromVenv("manage.py shell")
  474 + if not test_mode():
  475 + execProcessFromVenv("manage.py shell")
477 476 # Go back to the initial dir
478   - #os.chdir("../")
  477 + # os.chdir("../")
479 478 change_dir("PREVIOUS")
480 479 return True
481 480  
... ... @@ -489,7 +488,8 @@ def dbshell():
489 488 print("Type 'exit' to quit")
490 489 print()
491 490 # execProcess("python install.py install")
492   - if not test_mode(): execProcessFromVenv(PYROS_DJANGO_BASE_DIR+"/manage.py dbshell")
  491 + if not test_mode():
  492 + execProcessFromVenv(PYROS_DJANGO_BASE_DIR+"/manage.py dbshell")
493 493 # Go back to the initial dir
494 494 return True
495 495  
... ... @@ -497,22 +497,27 @@ def dbshell():
497 497 @pyros_launcher.command(help="Install the pyros software")
498 498 @click.option('--packages_only', '-p', is_flag=True, help='install only the python packages (no database installation)')
499 499 @click.option('--database_only', '-d', is_flag=True, help='install only the pyros database (no python packages installation)')
500   -#@global_test_options
  500 +# @global_test_options
501 501 def install(packages_only, database_only):
502   - install_or_update(UPDATE=False, packages_only=packages_only, database_only=database_only)
  502 + install_or_update(UPDATE=False, packages_only=packages_only,
  503 + database_only=database_only)
503 504  
504   -def install_or_update(UPDATE:bool=False, packages_only:bool=False, database_only:bool=False):
  505 +
  506 +def install_or_update(UPDATE: bool = False, packages_only: bool = False, database_only: bool = False):
505 507 SQL_USER = os.environ.get("MYSQL_PYROS_LOGIN").strip()
506 508 SQL_PSWD = os.environ.get("MYSQL_PYROS_PWD").strip()
507   - os.environ["PATH_TO_OBSCONF_FOLDER"] = os.path.join(os.path.abspath(PYROS_DJANGO_BASE_DIR),"../../../privatedev/config/default/")
508   - ACTION = "UPDATING" if UPDATE else "INSTALLING"
  509 + os.environ["PATH_TO_OBSCONF_FOLDER"] = os.path.join(os.path.abspath(
  510 + PYROS_DJANGO_BASE_DIR), "../../../privatedev/config/default/")
  511 + ACTION = "UPDATING" if UPDATE else "INSTALLING"
  512 +
509 513 if WITH_DOCKER:
510 514 database_only = True
511 515 if not packages_only and not database_only:
512 516 packages_only = database_only = True
513   - if UPDATE:
  517 +
  518 + if UPDATE:
514 519 print("Running UPDATE command")
515   - else:
  520 + else:
516 521 print("Running INSTALL command")
517 522 print("- packages_only:", packages_only)
518 523 print("- database_only:", database_only)
... ... @@ -521,17 +526,20 @@ def install_or_update(UPDATE:bool=False, packages_only:bool=False, database_only
521 526 from git import Repo
522 527 except:
523 528 pip = "pip" if IS_WINDOWS else "pip3"
524   - process = subprocess.Popen(pip + " install --upgrade GitPython", shell=True)
  529 + process = subprocess.Popen(
  530 + pip + " install --upgrade GitPython", shell=True)
525 531 process.wait()
526 532 if process.returncode == 0:
527   - # self.addExecuted(self.current_command, command)
  533 + # self.addExecuted(self.current_command, command)
528 534 from git import Repo
529 535 else:
530   - log.error("GitPython package (required for obsconfig class) installation failed")
  536 + log.error(
  537 + "GitPython package (required for obsconfig class) installation failed")
531 538  
532   - if not os.path.exists( os.path.join(os.getcwd(),"./vendor/guitastro")):
  539 + if not os.path.exists(os.path.join(os.getcwd(), "./vendor/guitastro")):
533 540 print("Cloning Guiastro repository")
534   - cloned_repo = Repo.clone_from("https://gitlab.irap.omp.eu/aklotz/guitastro.git", os.path.join(os.getcwd(),"./vendor/guitastro"))
  541 + cloned_repo = Repo.clone_from(
  542 + "https://gitlab.irap.omp.eu/aklotz/guitastro.git", os.path.join(os.getcwd(), "./vendor/guitastro"))
535 543 print("Cloned successfully: ", cloned_repo.__class__ is Repo)
536 544 #if test_mode(): print("in test mode")
537 545 # self.execProcess("python3 install/install.py install")
... ... @@ -540,63 +548,75 @@ def install_or_update(UPDATE:bool=False, packages_only:bool=False, database_only
540 548 num = 0
541 549 # 1) Update source code (git pull)
542 550 if UPDATE:
543   - num+=1
544   - printFullTerm(Colors.BLUE, f"{num}) UPDATING SOURCE CODE: Running git pull")
  551 + num += 1
  552 + printFullTerm(
  553 + Colors.BLUE, f"{num}) UPDATING SOURCE CODE: Running git pull")
545 554 if not WITH_DOCKER:
546 555 _gitpull() or die()
547 556  
548 557 # 2) Update python packages (pip upgrade AND pip install requirements)
549 558 if packages_only:
550   - num+=1
  559 + num += 1
551 560 printFullTerm(Colors.BLUE, f"{num}) {ACTION} PYTHON PACKAGES")
552 561 # (UPDATE) Re-install VENV if disappeared
553   - install_venv(EVEN_IF_ALREADY_EXISTS = not UPDATE)
  562 + install_venv(EVEN_IF_ALREADY_EXISTS=not UPDATE)
554 563 install_packages()
555 564  
556 565 # 3) Update PlantUML diagrams
557   - num+=1
  566 + num += 1
558 567 printFullTerm(Colors.BLUE, f"{num}) UPDATING UML DIAGRAMS")
559 568 _update_plantuml_diags() or die()
560 569 print(os.getcwd())
561   -
562   - # 4) Install/Update database structure (make migrations + migrate)
563   - if database_only:
564   - num+=1
  570 +
  571 + # 4) Update Sphinx API doc
  572 + num += 1
  573 + printFullTerm(Colors.BLUE, f"{num}) UPDATING API DOC (with Sphinx)")
  574 + _update_api_doc() or die()
  575 + exit()
  576 +
  577 + # 5) Install/Update database structure (make migrations + migrate)
  578 + if database_only:
  579 + num += 1
565 580 printFullTerm(Colors.BLUE, f"{num}) {ACTION} DATABASE")
566   - if UPDATE:
  581 + if UPDATE:
567 582 _updatedb() or die()
568 583 else:
569   - res = install_database(VENV)
570   - if(res==0):
571   - print(f"You can connect to PyROS as '{SQL_USER}' user with the password '{SQL_PSWD}' ")
  584 + res = install_database(VENV)
  585 + if(res == 0):
  586 + print(
  587 + f"You can connect to PyROS as '{SQL_USER}' user with the password '{SQL_PSWD}' ")
572 588 return True
573 589  
574 590  
575   -
576   -
577 591 @pyros_launcher.command(help="Run some tests")
578 592 @click.option('--app', '-a', help='app name')
579 593 def test(app):
580 594 print("Running tests")
581   - os.environ["PATH_TO_OBSCONF_FOLDER"] = os.path.join(os.path.abspath(PYROS_DJANGO_BASE_DIR),"obsconfig/fixtures/")
  595 + os.environ["PATH_TO_OBSCONF_FOLDER"] = os.path.join(
  596 + os.path.abspath(PYROS_DJANGO_BASE_DIR), "obsconfig/fixtures/")
582 597 os.environ["unit_name"] = ""
583 598 configfile = 'config_pyros.yml'
584   - os.environ["pyros_config_file"] = os.path.join(os.path.abspath(PYROS_DJANGO_BASE_DIR),"../../../config/pyros/", configfile)
  599 + os.environ["pyros_config_file"] = os.path.join(os.path.abspath(
  600 + PYROS_DJANGO_BASE_DIR), "../../../config/pyros/", configfile)
585 601 #start_dir = os.getcwd()
586 602 if app == None:
587 603 #apps = ['obsconfig','scientific_program','common', 'scheduler', 'routine_manager', 'user_manager', 'alert_manager.tests.TestStrategyChange']
588 604 # Removing alert_manager, scheduler from tests
589   - apps = ['obsconfig',"scientific_program",'common', 'user_manager', 'routine_manager']
  605 + apps = ['obsconfig', "scientific_program",
  606 + 'common', 'user_manager', 'routine_manager']
590 607 else:
591   - os.environ["PATH_TO_OBSCONF_FILE"] = os.path.join(os.path.abspath(PYROS_DJANGO_BASE_DIR),"obsconfig/fixtures/observatory_configuration_ok_simple.yml")
  608 + os.environ["PATH_TO_OBSCONF_FILE"] = os.path.join(os.path.abspath(
  609 + PYROS_DJANGO_BASE_DIR), "obsconfig/fixtures/observatory_configuration_ok_simple.yml")
592 610 change_dir(PYROS_DJANGO_BASE_DIR)
593   - execProcessFromVenv('manage.py test --keep --noinput --parallel 4 ' + app) or die()
  611 + execProcessFromVenv(
  612 + 'manage.py test --keep --noinput --parallel 4 ' + app) or die()
594 613 change_dir("PREVIOUS")
595 614 return True
596 615 for app in apps:
597 616 if app != "obsconfig":
598 617 # for every module except obsconfig, should use the simple observatory configuration in order to run the website with a configuration that isn't the observatory configuration used for production
599   - os.environ["PATH_TO_OBSCONF_FILE"] = os.path.join(os.path.abspath(PYROS_DJANGO_BASE_DIR),"obsconfig/fixtures/observatory_configuration_ok_simple.yml")
  618 + os.environ["PATH_TO_OBSCONF_FILE"] = os.path.join(os.path.abspath(
  619 + PYROS_DJANGO_BASE_DIR), "obsconfig/fixtures/observatory_configuration_ok_simple.yml")
600 620 #_loaddata() or die()
601 621 change_dir(PYROS_DJANGO_BASE_DIR)
602 622 # Delete test_pyros database after tests
... ... @@ -608,6 +628,7 @@ def test(app):
608 628 # execProcess("python install.py install")
609 629 return True
610 630  
  631 +
611 632 @pyros_launcher.command(help="Run ALL tests")
612 633 def testall():
613 634 change_dir(PYROS_DJANGO_BASE_DIR)
... ... @@ -619,8 +640,6 @@ def testall():
619 640 return True
620 641  
621 642  
622   -
623   -
624 643 @pyros_launcher.command(help="Update (only if necessary) the python packages AND the source code AND the DB structure")
625 644 def update():
626 645 install_or_update(UPDATE=True)
... ... @@ -653,13 +672,16 @@ def update():
653 672 def _gitpull():
654 673 print("-- running git pull")
655 674 GIT = "git.exe" if IS_WINDOWS else "git"
656   - if not test_mode(): return execProcess(f"{GIT} pull")
  675 + if not test_mode():
  676 + return execProcess(f"{GIT} pull")
657 677 return True
658 678  
659   -#@pyros_launcher.command(help="Update the pyros database")
  679 +# @pyros_launcher.command(help="Update the pyros database")
  680 +
  681 +
660 682 def _updatedb():
661 683 print("-- update db (make migrations + migrate)")
662   - if not test_mode() :
  684 + if not test_mode():
663 685 _makemigrations() or die()
664 686 _migrate() or die()
665 687 return True
... ... @@ -668,7 +690,7 @@ def _updatedb():
668 690 @pyros_launcher.command(help="Update the pyros database and fill it with initial fixture data")
669 691 def initdb():
670 692 if not test_mode():
671   - #updatedb()
  693 + # updatedb()
672 694 res1 = _makemigrations()
673 695 res2 = _migrate()
674 696 res3 = _loaddata()
... ... @@ -677,15 +699,15 @@ def initdb():
677 699  
678 700  
679 701 @pyros_launcher.command(help="Launch an agent")
680   -#@global_test_options
  702 +# @global_test_options
681 703 @click.argument('agent')
682 704 @click.option('--configfile', '-c', help='the configuration file to be used')
683 705 @click.option('--observatory', '-o', help='the observatory name to be used')
684 706 @click.option('--unit', '-u', help='the unit name to be used')
685   -#@click.option('--format', '-f', type=click.Choice(['html', 'xml', 'text']), default='html', show_default=True)
686   -#@click.option('--port', default=8000)
687   -#def start(agent:str, configfile:str, test, verbosity):
688   -def start(agent:str, configfile:str, observatory:str, unit:str):
  707 +# @click.option('--format', '-f', type=click.Choice(['html', 'xml', 'text']), default='html', show_default=True)
  708 +# @click.option('--port', default=8000)
  709 +# def start(agent:str, configfile:str, test, verbosity):
  710 +def start(agent: str, configfile: str, observatory: str, unit: str):
689 711 log.debug("Running start command")
690 712 try:
691 713 from config.pyros.config_pyros import ConfigPyros
... ... @@ -693,32 +715,41 @@ def start(agent:str, configfile:str, observatory:str, unit:str):
693 715 #pip = "pip" if platform.system() == "Windows" else "pip3"
694 716 # TODO: "python -m pip" au lieu de "pip"
695 717 pip = "pip" if IS_WINDOWS else "pip3"
696   - process = subprocess.Popen(pip + " install --upgrade pykwalify", shell=True)
  718 + process = subprocess.Popen(
  719 + pip + " install --upgrade pykwalify", shell=True)
697 720 process.wait()
698 721 if process.returncode == 0:
699 722 # self.addExecuted(self.current_command, command)
700 723 from config.pyros.config_pyros import configpyros
701 724 else:
702   - log.error("pykwalify package (required for obsconfig class) installation failed")
703   - if configfile:
704   - log.debug("With config file"+ configfile)
705   - os.environ["pyros_config_file"] = os.path.join(os.path.abspath(PYROS_DJANGO_BASE_DIR),"../../../config/pyros/", configfile)
706   - else:
  725 + log.error(
  726 + "pykwalify package (required for obsconfig class) installation failed")
  727 + if configfile:
  728 + log.debug("With config file" + configfile)
  729 + os.environ["pyros_config_file"] = os.path.join(os.path.abspath(
  730 + PYROS_DJANGO_BASE_DIR), "../../../config/pyros/", configfile)
  731 + else:
707 732 configfile = 'config_pyros.yml'
708   - os.environ["pyros_config_file"] = os.path.join(os.path.abspath(PYROS_DJANGO_BASE_DIR),"../../../config/pyros/", configfile)
709   -
710   - logo_name = ConfigPyros(os.environ["pyros_config_file"]).pyros_config["general"]["logo"]
711   - logo_path = os.path.join(os.path.abspath(PYROS_DJANGO_BASE_DIR),"../../../config/pyros/", logo_name)
712   - media_path = os.path.join(os.path.abspath(PYROS_DJANGO_BASE_DIR),"misc/static/media",logo_name)
713   - shutil.copy(logo_path,media_path)
  733 + os.environ["pyros_config_file"] = os.path.join(os.path.abspath(
  734 + PYROS_DJANGO_BASE_DIR), "../../../config/pyros/", configfile)
  735 +
  736 + logo_name = ConfigPyros(
  737 + os.environ["pyros_config_file"]).pyros_config["general"]["logo"]
  738 + logo_path = os.path.join(os.path.abspath(
  739 + PYROS_DJANGO_BASE_DIR), "../../../config/pyros/", logo_name)
  740 + media_path = os.path.join(os.path.abspath(
  741 + PYROS_DJANGO_BASE_DIR), "misc/static/media", logo_name)
  742 + shutil.copy(logo_path, media_path)
714 743 #if test_mode(): print("in test mode")
715 744 #if verbose_mode(): print("in verbose mode")
716 745 if observatory == None or len(observatory) == 0:
717 746 observatory = "default"
718   - observatories_configuration_folder = os.path.join(os.path.abspath(PYROS_DJANGO_BASE_DIR),"../../../privatedev/config/")
  747 + observatories_configuration_folder = os.path.join(
  748 + os.path.abspath(PYROS_DJANGO_BASE_DIR), "../../../privatedev/config/")
719 749 if len(glob.glob(observatories_configuration_folder+observatory+"/")) != 1:
720 750 # Observatory configuration folder not found
721   - print(f"Observatory configuration folder for observatory '{observatory}' not found in {observatories_configuration_folder}")
  751 + print(
  752 + f"Observatory configuration folder for observatory '{observatory}' not found in {observatories_configuration_folder}")
722 753 print("Available observatories configuration :")
723 754 for obs_conf_folder in os.listdir(observatories_configuration_folder):
724 755 print(obs_conf_folder)
... ... @@ -726,10 +757,12 @@ def start(agent:str, configfile:str, observatory:str, unit:str):
726 757  
727 758 path_to_obs_config_folder = observatories_configuration_folder+observatory+"/"
728 759 obs_config_file_name = ""
729   - # Search for observatory config file
730   - obs_config_file_name = glob.glob(path_to_obs_config_folder+"/observatory*.yml")[0]
731   -
732   - obs_config_file_path = os.path.join(path_to_obs_config_folder,obs_config_file_name)
  760 + # Search for observatory config file
  761 + obs_config_file_name = glob.glob(
  762 + path_to_obs_config_folder+"/observatory*.yml")[0]
  763 +
  764 + obs_config_file_path = os.path.join(
  765 + path_to_obs_config_folder, obs_config_file_name)
733 766 os.environ["PATH_TO_OBSCONF_FILE"] = obs_config_file_path
734 767 os.environ["PATH_TO_OBSCONF_FOLDER"] = path_to_obs_config_folder
735 768 os.environ["unit_name"] = unit if unit else ''
... ... @@ -739,17 +772,18 @@ def start(agent:str, configfile:str, observatory:str, unit:str):
739 772 else:
740 773 os.environ["unit_name"] = ""
741 774 '''
742   -
  775 +
743 776 # add path to pyros_django folder as the config class is supposed to work within this folder
744 777 #cmd_test_obs_config = f"-c \"from src.core.pyros_django.obsconfig.obsconfig_class import OBSConfig\nOBSConfig('{os.path.join(PYROS_DJANGO_BASE_DIR,os.environ.get('PATH_TO_OBSCONF_FILE'))}')\""
745 778 cmd_test_obs_config = f"-c \"from src.core.pyros_django.obsconfig.obsconfig_class import OBSConfig\nOBSConfig('{obs_config_file_path}')\""
746 779 if not execProcessFromVenv(cmd_test_obs_config):
747 780 # Observatory configuration has an issue
748   - exit(1)
749   - # Check each agent in the given list
  781 + exit(1)
  782 + # Check each agent in the given list
750 783 agents = agent.split(",") if "," in agent else [agent]
751   - for a in agents:
752   - if not _check_agent(a): return
  784 + for a in agents:
  785 + if not _check_agent(a):
  786 + return
753 787 #print("Agents are:", agents)
754 788 '''
755 789 # Check if multiple agents:
... ... @@ -764,15 +798,15 @@ def start(agent:str, configfile:str, observatory:str, unit:str):
764 798 agents = [agent]
765 799 if not _check_agent(agent): return
766 800 '''
767   -
  801 +
768 802 # Start Agents (processes)
769 803 current_processes = []
770   - for agent_name,agent_folder in AGENTS.items():
  804 + for agent_name, agent_folder in AGENTS.items():
771 805  
772   - #if agent in ("all", agent_name) :
773   - if agent=="all" or agent_name in agents:
  806 + # if agent in ("all", agent_name) :
  807 + if agent == "all" or agent_name in agents:
774 808 # Default case, launch agentX
775   - #if agent_name == "agentX":
  809 + # if agent_name == "agentX":
776 810  
777 811 # execProcessFromVenvAsync(VENV_PYTHON + " manage.py runserver")
778 812 log.debug(VENV_PYTHON)
... ... @@ -783,22 +817,22 @@ def start(agent:str, configfile:str, observatory:str, unit:str):
783 817  
784 818 # OLD format agents: majordome, monitoring, alert...
785 819 cmd = "start_agent.py " + agent_name + " " + configfile
786   -
  820 +
787 821 # Agent "webserver"
788   - if agent_name == "webserver":
  822 + if agent_name == "webserver":
789 823 cmd = "manage.py runserver"
790 824 if(WITH_DOCKER):
791 825 # If we're running pyros within docker, we need to specify a specific adress in order to access the website on our host machine
792 826 PYROS_WEBSITE_PORT = os.environ.get("PYROS_WEBSITE_PORT")
793   - cmd=f"manage.py runserver 0.0.0.0:{PYROS_WEBSITE_PORT}"
  827 + cmd = f"manage.py runserver 0.0.0.0:{PYROS_WEBSITE_PORT}"
794 828 os.chdir(PYROS_DJANGO_BASE_DIR)
795 829 #if not test_mode(): execProcessFromVenv("start_agent.py " + agent_name + " " + configfile)
796 830  
797   - # Agent "agentM", "agentA", "agentB", "agentX", ...
  831 + # Agent "agentM", "agentA", "agentB", "agentX", ...
798 832 elif agent_name.startswith("agent"):
799 833 # Run agent without actual commands sent to devices (FOR_REAL=False)
800   - ##agentX.run(FOR_REAL=True)
801   - if agent_name == "agentM":
  834 + # agentX.run(FOR_REAL=True)
  835 + if agent_name == "agentM":
802 836 os.chdir(PYROS_DJANGO_BASE_DIR+"/monitoring/")
803 837 elif agent_name == "agentSP":
804 838 os.chdir(PYROS_DJANGO_BASE_DIR+"/scientific_program/")
... ... @@ -807,53 +841,62 @@ def start(agent:str, configfile:str, observatory:str, unit:str):
807 841 #cmd = "-m AgentX"
808 842 #cmd = f" Agent{agent_name[5:]}.py {configfile}"
809 843 cmd = f"Agent{agent_name[5:]}.py"
810   - if debug_mode(): cmd += " -d"
811   - if sim_mode(): cmd += " -s"
812   - if test_mode(): cmd += " -t"
813   - if verbose_mode(): cmd += " -v"
814   - if configfile: cmd += " {configfile}"
815   -
  844 + if debug_mode():
  845 + cmd += " -d"
  846 + if sim_mode():
  847 + cmd += " -s"
  848 + if test_mode():
  849 + cmd += " -t"
  850 + if verbose_mode():
  851 + cmd += " -v"
  852 + if configfile:
  853 + cmd += " {configfile}"
  854 +
816 855 #if not test_mode(): current_processes.append( [execProcessFromVenvAsync(cmd), agent_name, -1] )
817 856 # Append this process ( [process id, agent_name, result=failure] )
818 857 # ("result" will be updated at the end of execution)
819   - current_processes.append( [execProcessFromVenvAsync(cmd), agent_name, -1] )
  858 + current_processes.append(
  859 + [execProcessFromVenvAsync(cmd), agent_name, -1])
820 860 # self.change_dir("..")
821 861 os.chdir(current_dir)
822 862  
823 863 # Go back to root folder (/)
824 864 # self.change_dir('..')
825   - #os.chdir("..")
  865 + # os.chdir("..")
826 866 # Wait for end of each process execution
827   - #for (p,agent) in current_processes:
  867 + # for (p,agent) in current_processes:
828 868 for process in current_processes:
829   - p,agent,_ = process
830   - log.debug(f"************ Waiting for end of execution of agent {agent} ************")
  869 + p, agent, _ = process
  870 + log.debug(
  871 + f"************ Waiting for end of execution of agent {agent} ************")
831 872 p.wait()
832 873 process[2] = p.returncode
833 874 print(f"************ END of execution of agent {agent} ************")
834 875 if p.returncode == 0:
835   - printFullTerm(Colors.GREEN, f"Process {agent} executed successfully")
  876 + printFullTerm(
  877 + Colors.GREEN, f"Process {agent} executed successfully")
836 878 # self.addExecuted(self.current_command, command)
837 879 else:
838 880 printFullTerm(Colors.WARNING, f"Process {agent} execution failed")
839 881 # self.addError(self.current_command, command)
840   -
  882 +
841 883 print()
842 884 print()
843 885 print("Synthesis of the results:")
844 886 for process in current_processes:
845   - p,agent,returncode = process
  887 + p, agent, returncode = process
846 888 if returncode == 0:
847   - printFullTerm(Colors.GREEN, f"Process {agent} executed successfully")
  889 + printFullTerm(
  890 + Colors.GREEN, f"Process {agent} executed successfully")
848 891 # self.addExecuted(self.current_command, command)
849 892 else:
850 893 printFullTerm(Colors.WARNING, f"Process {agent} execution failed")
851 894 # self.addError(self.current_command, command)
852   -
  895 +
853 896 #print("************ end of START() ************")
854 897 # Only according to the last process status:
855   - #return True if p.returncode==0 else False
856   - return True if p.returncode==0 else False
  898 + # return True if p.returncode==0 else False
  899 + return True if p.returncode == 0 else False
857 900  
858 901  
859 902 # TODO: implémenter le STOP !!!
... ... @@ -861,8 +904,8 @@ def start(agent:str, configfile:str, observatory:str, unit:str):
861 904 @click.argument('agent')
862 905 def stop(agent):
863 906 print("Running stop command")
864   - if not _check_agent(agent): return
865   -
  907 + if not _check_agent(agent):
  908 + return
866 909  
867 910  
868 911 """
... ... @@ -871,14 +914,17 @@ def stop(agent):
871 914 ********************************************************************************
872 915 """
873 916  
  917 +
874 918 def notused_update_python_packages_from_requirements():
875 919 # 1) Upgrade pip (if new version available)
876 920 res = execProcessFromVenv("-m pip install --upgrade pip")
877   - if not res: return False
  921 + if not res:
  922 + return False
878 923 # 2) Install only "not yet installed" python packages
879 924 res = execProcessFromVenv("-m pip install -r install/"+REQUIREMENTS)
880 925 return res
881 926  
  927 +
882 928 def _update_plantuml_diags():
883 929 for dirpath, dirnames, files in os.walk(PYROS_DJANGO_BASE_DIR):
884 930 if os.path.basename(dirpath) == "doc":
... ... @@ -887,13 +933,34 @@ def _update_plantuml_diags():
887 933 for diag in diagrams:
888 934 # if no diag.png or if diag.pu more recent than diag.png => re-generate diag.png
889 935 diag_png = diag.split('.pu')[0] + '.png'
890   - if not os.path.isfile(diag_png) or ( os.path.getmtime(diag) > os.path.getmtime(diag_png) ) :
  936 + if not os.path.isfile(diag_png) or (os.path.getmtime(diag) > os.path.getmtime(diag_png)):
891 937 #res = execProcessFromVenv("-m plantuml "+diag)
892 938 # we need to use our own version of plantuml
893 939 res = execProcessFromVenv("./install/plantuml.py "+diag)
894   - if not res: return False
  940 + if not res:
  941 + return False
895 942 return True
896 943  
  944 +
  945 +def _update_api_doc():
  946 + print(os.getcwd())
  947 + DOC_RST_PATH = "doc/doc_rst/"
  948 + # 1) install/update Sphinx requirements (only if not yet installed)
  949 + venv_pip_install2(DOC_RST_PATH+'requirements.txt', '-r')
  950 + # 2) make html doc from RST doc
  951 + # cd doc/sourcedoc/
  952 + change_dir(DOC_RST_PATH)
  953 + # ./make_rst_then_html
  954 + res = execProcess('make clean ; /bin/bash make_rst_then_html')
  955 + print("The technical (and API) documentation has been generated and can be opened in a local browser with this command :")
  956 + print(" open "+DOC_RST_PATH+"build/html/index.html")
  957 + print(" (or just open your local browser _firefox, chrome, ..._ and go to this file)")
  958 + # Come back to where we were before
  959 + # cd -
  960 + change_dir("PREVIOUS")
  961 + return True
  962 +
  963 +
897 964 def _migrate():
898 965 change_dir(PYROS_DJANGO_BASE_DIR)
899 966 # Migrate only migrations for the app "common"
... ... @@ -903,6 +970,7 @@ def _migrate():
903 970 change_dir("PREVIOUS")
904 971 return res
905 972  
  973 +
906 974 def _makemigrations():
907 975 change_dir(PYROS_DJANGO_BASE_DIR)
908 976 #execProcessFromVenv(self.venv_bin + " manage.py makemigrations")
... ... @@ -911,52 +979,54 @@ def _makemigrations():
911 979 change_dir("PREVIOUS")
912 980 return res
913 981  
914   -#TODO: mettre la fixture en date naive (sans time zone)
  982 +# TODO: mettre la fixture en date naive (sans time zone)
  983 +
  984 +
915 985 def _loaddata():
916 986 change_dir(PYROS_DJANGO_BASE_DIR)
917 987 #execProcessFromVenv(self.venv_bin + " manage.py loaddata misc" + os.sep + "fixtures" + os.sep + self.INIT_FIXTURE)
918   - res = execProcessFromVenv("manage.py loaddata misc" + os.sep + "fixtures" + os.sep + INIT_FIXTURE)
  988 + res = execProcessFromVenv(
  989 + "manage.py loaddata misc" + os.sep + "fixtures" + os.sep + INIT_FIXTURE)
919 990 change_dir("PREVIOUS")
920 991 return res
921 992  
922 993  
923 994 def change_dir(path):
924 995 global _previous_dir
925   - if path == "PREVIOUS": path=_previous_dir
  996 + if path == "PREVIOUS":
  997 + path = _previous_dir
926 998 _previous_dir = os.getcwd()
927 999 log.debug("Moving to : " + path)
928 1000 os.chdir(path)
929 1001 log.debug("Current directory : " + str(os.getcwd()))
930 1002  
931 1003  
932   -
933 1004 def _check_agent(agent):
934 1005 # Check that agent exists
935   - if agent != "all" and agent not in AGENTS.keys() :
  1006 + if agent != "all" and agent not in AGENTS.keys():
936 1007 # Check if multiple agents:
937 1008 print("This agent does not exist")
938 1009 print("Here is the allowed list of agents:")
939 1010 print("- all => will launch ALL agents")
940   - for agent_name in AGENTS.keys(): print('-',agent_name)
  1011 + for agent_name in AGENTS.keys():
  1012 + print('-', agent_name)
941 1013 return False
942 1014 return True
943 1015  
944 1016  
945   -
946   -
947   -
948   -
949 1017 """
950 1018 INSTALLATION FUNCTIONS (./pyros.py install)
951 1019 """
952 1020  
  1021 +
953 1022 def replacePatternInFile(pattern, replace, file_path):
954 1023 try:
955 1024 with fileinput.FileInput(file_path, inplace=True, backup='.bak') as file:
956 1025 for line in file:
957 1026 print(line.replace(pattern, replace), end='')
958 1027 except:
959   - sys.stderr.write(Colors.ERROR + "ERROR !: replacement in file failed !" + Colors.END + "\r\n")
  1028 + sys.stderr.write(
  1029 + Colors.ERROR + "ERROR !: replacement in file failed !" + Colors.END + "\r\n")
960 1030 return 1
961 1031 return 0
962 1032  
... ... @@ -975,7 +1045,8 @@ def notused_install_dependency_ubuntu(command, mode):
975 1045 process = subprocess.Popen(command, shell=True)
976 1046 process.wait()
977 1047 if process.returncode != 0:
978   - sys.stderr.write(Colors.ERROR + "ERROR !: installation of " + old + " failed !" + Colors.END + "\r\n")
  1048 + sys.stderr.write(Colors.ERROR + "ERROR !: installation of " +
  1049 + old + " failed !" + Colors.END + "\r\n")
979 1050  
980 1051  
981 1052 def notused_install_required_ubuntu():
... ... @@ -998,7 +1069,8 @@ def notused_install_dependency_centos(command, mode):
998 1069 process = subprocess.Popen(command, shell=True)
999 1070 process.wait()
1000 1071 if process.returncode != 0:
1001   - sys.stderr.write(Colors.ERROR + "ERROR !: installation of " + old + " failed !" + Colors.END + "\r\n")
  1072 + sys.stderr.write(Colors.ERROR + "ERROR !: installation of " +
  1073 + old + " failed !" + Colors.END + "\r\n")
1002 1074  
1003 1075  
1004 1076 def notused_install_required_centos():
... ... @@ -1034,7 +1106,8 @@ def notused_install_required():
1034 1106 # Find the linux distribution and call the related function
1035 1107 distribution = platform.dist()
1036 1108 if not 'SUDO_UID' in os.environ.keys():
1037   - sys.stderr.write("Super user rights are needed to install prerequisites\r\n")
  1109 + sys.stderr.write(
  1110 + "Super user rights are needed to install prerequisites\r\n")
1038 1111 exit(1)
1039 1112 if distribution[0] == "Ubuntu" or distribution[0] == "Debian":
1040 1113 install_required_ubuntu()
... ... @@ -1045,11 +1118,16 @@ def notused_install_required():
1045 1118 exit(1)
1046 1119  
1047 1120  
1048   -def venv_pip_install(package_name:str, options:str=''):
  1121 +def venv_pip_install(package_name: str, options: str = ''):
1049 1122 os.system(VENV_PIP + ' install ' + options + ' ' + package_name)
1050 1123  
1051 1124  
1052   -def install_venv(EVEN_IF_ALREADY_EXISTS:bool=False):
  1125 +def venv_pip_install2(package_name: str, options: str = ''):
  1126 + os.system(VENV_PYTHON + ' -m ' + VENV_PIP + ' install '
  1127 + + options + ' ' + package_name)
  1128 +
  1129 +
  1130 +def install_venv(EVEN_IF_ALREADY_EXISTS: bool = False):
1053 1131 '''
1054 1132 Install VENV if does not exist
1055 1133 OR
... ... @@ -1072,14 +1150,14 @@ def install_venv(EVEN_IF_ALREADY_EXISTS:bool=False):
1072 1150 # --- (If not exists) Create the (private) venv ROOT directory (where the virtual environment dir will be created)
1073 1151 # --------------------------------------------
1074 1152 if (os.path.basename(os.getcwd()) != VENV_ROOT):
1075   - ##if not(os.path.isdir("../venv")):
  1153 + # if not(os.path.isdir("../venv")):
1076 1154 if not(os.path.isdir(VENV_ROOT)):
1077   - print(Colors.LOG_BLUE + f"-----------------------------Creating 'venv' root directory ({VENV_ROOT}/)-----------------------------" + Colors.END)
1078   - #os.mkdir("../venv")
  1155 + print(Colors.LOG_BLUE +
  1156 + f"-----------------------------Creating 'venv' root directory ({VENV_ROOT}/)-----------------------------" + Colors.END)
  1157 + # os.mkdir("../venv")
1079 1158 os.mkdir(VENV_ROOT)
1080 1159 os.chdir(VENV_ROOT)
1081 1160  
1082   -
1083 1161 # --------------------------------------------
1084 1162 # --- (if EVEN_IF_ALREADY_EXISTS) Delete venv dir if already exist
1085 1163 # --------------------------------------------
... ... @@ -1088,20 +1166,22 @@ def install_venv(EVEN_IF_ALREADY_EXISTS:bool=False):
1088 1166 while True:
1089 1167 try:
1090 1168 if os.path.isdir(VENV):
1091   - print(Colors.LOG_BLUE + f"-----------------------------Deleting existing VENV dir ({VENV}/)-----------------------------" + Colors.END)
  1169 + print(
  1170 + Colors.LOG_BLUE + f"-----------------------------Deleting existing VENV dir ({VENV}/)-----------------------------" + Colors.END)
1092 1171 shutil.rmtree(VENV)
1093 1172 break
1094 1173 # Exception on Windows WinError 145 : Cannot remove folder because files in folder not yet removed...
1095 1174 except Exception as e:
1096   - #print(e)
  1175 + # print(e)
1097 1176 continue
1098 1177  
1099 1178 # --------------------------------------------
1100 1179 # --- (if not exists) Reinstall the virtual environment (from ../venv/)
1101 1180 # --------------------------------------------
1102   -
  1181 +
1103 1182 if not os.path.isdir(VENV):
1104   - print(Colors.LOG_BLUE + f"-----------------------------Creating venv dir ({VENV}/)-----------------------------"+END_OF_LINE + Colors.END)
  1183 + print(Colors.LOG_BLUE +
  1184 + f"-----------------------------Creating venv dir ({VENV}/)-----------------------------"+END_OF_LINE + Colors.END)
1105 1185 os.system(GLOBAL_PYTHON+" -m venv " + VENV)
1106 1186  
1107 1187 # Come back to project root
... ... @@ -1113,7 +1193,8 @@ def install_packages():
1113 1193  
1114 1194 os.chdir(VENV_ROOT)
1115 1195  
1116   - print(Colors.LOG_BLUE + "-----------------------------Upgrade pip, wheel, and setuptools" + "-----------------------------"+END_OF_LINE + Colors.END)
  1196 + print(Colors.LOG_BLUE + "-----------------------------Upgrade pip, wheel, and setuptools" +
  1197 + "-----------------------------"+END_OF_LINE + Colors.END)
1117 1198 # Upgrade pip
1118 1199 os.system(VENV_PYTHON + ' -m pip install --upgrade pip')
1119 1200 '''
... ... @@ -1122,14 +1203,14 @@ def install_packages():
1122 1203 else: # Linux
1123 1204 os.system(venv + '/bin/python -m pip install --upgrade pip')
1124 1205 '''
1125   -
  1206 +
1126 1207 # Pip upgrade wheel and setuptools
1127 1208 venv_pip_install('wheel', '--upgrade')
1128 1209 #os.system(VENV_PIP+' install --upgrade wheel')
1129 1210 # Not working with python 3.8 (on 17/02/2022)
1130 1211 # venv_pip_install('setuptools', '--upgrade')
1131 1212 os.system(VENV_PIP+' install setuptools==58')
1132   -
  1213 +
1133 1214 # Pip install required packages from REQUIREMENTS file
1134 1215 print()
1135 1216 print(Colors.LOG_BLUE + "-----------------------------Installing python packages via pip-----------------------------" + Colors.END)
... ... @@ -1140,11 +1221,12 @@ def install_packages():
1140 1221  
1141 1222 if IS_WINDOWS:
1142 1223 os.chdir("../install")
1143   - ## moving voeventparse in site-packages directory
  1224 + # moving voeventparse in site-packages directory
1144 1225 try:
1145 1226 site_packages = "..\\venv\\"+VENV+"\\Lib\\site-packages\\"
1146 1227 if (
1147   - not os.path.isdir(site_packages + "voevent_parse-0.9.5.dist-info")
  1228 + not os.path.isdir(
  1229 + site_packages + "voevent_parse-0.9.5.dist-info")
1148 1230 and
1149 1231 not os.path.isdir(site_packages + "voeventparse")
1150 1232 ):
... ... @@ -1152,19 +1234,24 @@ def install_packages():
1152 1234 cmdline = "xcopy /i /y windows\\voeventparse " + site_packages + "voeventparse"
1153 1235 process = subprocess.Popen(cmdline)
1154 1236 process.wait()
1155   - if (process.returncode != 0): raise Exception
1156   - process = subprocess.Popen("xcopy /i /y windows\\voevent_parse-0.9.5.dist-info " + site_packages + "voevent_parse-0.9.5.dist-info")
  1237 + if (process.returncode != 0):
  1238 + raise Exception
  1239 + process = subprocess.Popen(
  1240 + "xcopy /i /y windows\\voevent_parse-0.9.5.dist-info " + site_packages + "voevent_parse-0.9.5.dist-info")
1157 1241 process.wait()
1158   - if (process.returncode != 0): raise Exception
  1242 + if (process.returncode != 0):
  1243 + raise Exception
1159 1244 print(Colors.LOG_BLUE + "\r\n-----------------------------library successfully copied-----------------------------" + Colors.END)
1160 1245 except Exception as e:
1161   - print(Colors.ERROR + "ERROR while Copying the voevent library in Lib/site-packages" + Colors.END) ; #, file=stderr)
  1246 + # , file=stderr)
  1247 + print(
  1248 + Colors.ERROR + "ERROR while Copying the voevent library in Lib/site-packages" + Colors.END)
1162 1249 return False
1163   -
  1250 +
1164 1251 # Go back to project root dir
1165 1252 os.chdir('..')
1166   -
1167   - #return 0
  1253 +
  1254 + # return 0
1168 1255 return True
1169 1256  
1170 1257  
... ... @@ -1172,9 +1259,10 @@ def install_database(venv):
1172 1259  
1173 1260 SQL_USER = os.environ.get("MYSQL_PYROS_LOGIN").strip()
1174 1261 SQL_PSWD = os.environ.get("MYSQL_PYROS_PWD").strip()
1175   -
1176   - print(Colors.LOG_BLUE + END_OF_LINE+"-----------------------------Launching mysql to create database and create and grant user pyros-----------------------------" + Colors.END)
1177   -
  1262 +
  1263 + print(Colors.LOG_BLUE + END_OF_LINE +
  1264 + "-----------------------------Launching mysql to create database and create and grant user pyros-----------------------------" + Colors.END)
  1265 +
1178 1266 # --------------------------------------------
1179 1267 # --- Determine the MySQL version
1180 1268 # --------------------------------------------
... ... @@ -1211,9 +1299,9 @@ def install_database(venv):
1211 1299 f"DROP DATABASE {IF_EXISTS} {SQL_DATABASE_SIMU}; CREATE DATABASE {SQL_DATABASE_SIMU}; " +\
1212 1300 f"DROP DATABASE {IF_EXISTS} {SQL_DATABASE_TEST}; CREATE DATABASE {SQL_DATABASE_TEST}; "
1213 1301  
1214   - # 2) Create user pyros and give it all rights on 3 databases
  1302 + # 2) Create user pyros and give it all rights on 3 databases
1215 1303 # Ne marche pas si l'utilisateur existe déjà => erreur
1216   - #"CREATE USER "+SQL_USER+"; " +\
  1304 + # "CREATE USER "+SQL_USER+"; " +\
1217 1305 # Donc, il faut ruser
1218 1306 # Mysql >= 5.7 only:
1219 1307 #sql_query_elem = "CREATE USER IF NOT EXISTS "+SQL_USER+"; "
... ... @@ -1227,7 +1315,7 @@ def install_database(venv):
1227 1315 else:
1228 1316 host = "localhost"
1229 1317 # FOR MYSQL 8:
1230   - if sql_version >=8:
  1318 + if sql_version >= 8:
1231 1319 # We remove identified by because grant was creating the pyros user, wasn't working on version 8 so we separated the steps :
1232 1320 # first we create the user then we grant him rights
1233 1321 sql_query += f"CREATE USER IF NOT EXISTS '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; "
... ... @@ -1237,7 +1325,7 @@ def install_database(venv):
1237 1325 else:
1238 1326 sql_query += f"GRANT ALL ON {SQL_DATABASE}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; "
1239 1327 sql_query += f"GRANT ALL ON {SQL_DATABASE_SIMU}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; "
1240   - # This database does not yet exists and will be automatically created by Django, but we already give access rights to it for pyros user
  1328 + # This database does not yet exists and will be automatically created by Django, but we already give access rights to it for pyros user
1241 1329 sql_query += f"GRANT ALL ON {SQL_DATABASE_TEST}.* TO '{SQL_USER}'@'{host}' IDENTIFIED BY '{SQL_PSWD}'; "
1242 1330 # FOR MYSQL 8:
1243 1331 if sql_version >= 8:
... ... @@ -1259,12 +1347,12 @@ def install_database(venv):
1259 1347 #req = "drop database pyros ; CREATE DATABASE pyros; drop database pyros_test ; CREATE DATABASE pyros_test; DROP USER 'pyros'@'localhost' ; GRANT USAGE ON *.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros' WITH GRANT OPTION; DROP USER 'pyros'@'localhost'; GRANT ALL ON pyros.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros'; GRANT ALL PRIVILEGES ON pyros.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros' WITH GRANT OPTION; GRANT ALL PRIVILEGES ON pyros_test.* TO pyros@localhost IDENTIFIED BY 'DjangoPyros' WITH GRANT OPTION;"
1260 1348 # (EP) ok for CENTOS 7 I suppose (but not for CentOS 6):
1261 1349 #req_centos = "CREATE DATABASE IF NOT EXISTS pyros; CREATE DATABASE IF NOT EXISTS pyros_test; GRANT USAGE ON *.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros' WITH GRANT OPTION; DROP USER 'pyros'@'localhost'; GRANT ALL ON pyros.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros'; GRANT ALL PRIVILEGES ON pyros.* TO 'pyros'@'localhost' IDENTIFIED BY 'DjangoPyros' WITH GRANT OPTION; GRANT ALL PRIVILEGES ON pyros_test.* TO pyros@localhost IDENTIFIED BY 'DjangoPyros' WITH GRANT OPTION;"
1262   -
  1350 +
1263 1351 # --- Prepare the SQL query to create and initialize the pyros database if needed
1264   - #if platform.dist()[0] == "centos":
  1352 + # if platform.dist()[0] == "centos":
1265 1353 # req = sql_query
1266 1354 #mysql_call_root = '"' + MYSQL_EXE_PATH + 'mysql" -u root -p'
1267   -
  1355 +
1268 1356 # if the mysql_root_password isn't defined, it will ask to enter the password
1269 1357 try:
1270 1358 # a root password should be defined as an environment variable
... ... @@ -1284,48 +1372,52 @@ def install_database(venv):
1284 1372  
1285 1373 # when using -p option, there is no space between the option and the password value
1286 1374 mysql_call_root = f"mysql -h {mysql_host} -u '{mysql_login}' -p'{mysql_root_password}'"
1287   - mysql_call_pyros = "\"" + MYSQL_EXE_PATH+ "mysql\" -u "+SQL_USER+" -p"
1288   -
  1375 + mysql_call_pyros = "\"" + MYSQL_EXE_PATH + "mysql\" -u "+SQL_USER+" -p"
  1376 +
1289 1377 # --------------------------------------------
1290 1378 # --- Creating database and creating and granting user pyros
1291 1379 # --------------------------------------------
1292 1380 user_ros_is_created = True
1293   - if sql_version<5.5:
1294   - print(Colors.LOG_BLUE +"------------------ Check if the user pyros exists in MYSQL (type the pyros password) -----------------------------" + Colors.END)
1295   - # --- We are testing if user pyros already exists in the database
1296   - process = subprocess.Popen("echo quit |" + mysql_call_pyros, shell=True)
1297   - process.wait()
1298   - if (process.returncode == 0):
1299   - user_ros_is_created = False
  1381 + if sql_version < 5.5:
  1382 + print(Colors.LOG_BLUE + "------------------ Check if the user pyros exists in MYSQL (type the pyros password) -----------------------------" + Colors.END)
  1383 + # --- We are testing if user pyros already exists in the database
  1384 + process = subprocess.Popen(
  1385 + "echo quit |" + mysql_call_pyros, shell=True)
  1386 + process.wait()
  1387 + if (process.returncode == 0):
  1388 + user_ros_is_created = False
1300 1389 if user_ros_is_created:
1301   - # --- The user pyros must be created in the database
1302   - print(Colors.LOG_BLUE +"-----------------------------Please enter your MYSQL root password-----------------------------" + Colors.END)
1303   - #process = subprocess.Popen("echo \"" + sql_query + "\" |"+ mysql_call_root, shell=True)
1304   - sql_cmd = 'echo "' + sql_query + '" | '+ mysql_call_root
1305   - print("Executing sql cmd: ", sql_cmd)
1306   - process = subprocess.Popen(sql_cmd, shell=True)
1307   - process.wait()
  1390 + # --- The user pyros must be created in the database
  1391 + print(Colors.LOG_BLUE + "-----------------------------Please enter your MYSQL root password-----------------------------" + Colors.END)
  1392 + #process = subprocess.Popen("echo \"" + sql_query + "\" |"+ mysql_call_root, shell=True)
  1393 + sql_cmd = 'echo "' + sql_query + '" | ' + mysql_call_root
  1394 + print("Executing sql cmd: ", sql_cmd)
  1395 + process = subprocess.Popen(sql_cmd, shell=True)
  1396 + process.wait()
1308 1397 if (process.returncode != 0):
1309   - sys.stderr.write(Colors.ERROR + "ERROR !: db configuration failed !" + Colors.END + "\r\n")
  1398 + sys.stderr.write(
  1399 + Colors.ERROR + "ERROR !: db configuration failed !" + Colors.END + "\r\n")
1310 1400 return -1
1311   - print(Colors.LOG_BLUE + END_OF_LINE+"-----------------------------Database created and user pyros successfully created and granted-----------------------------" + Colors.END)
  1401 + print(Colors.LOG_BLUE + END_OF_LINE +
  1402 + "-----------------------------Database created and user pyros successfully created and granted-----------------------------" + Colors.END)
1312 1403  
1313 1404 # --------------------------------------------
1314 1405 # --- Replacing pattern in settings.py to use mysql
1315 1406 # --------------------------------------------
1316 1407 print(Colors.LOG_BLUE + "-----------------------------setting MYSQL = True in settings-----------------------------" + Colors.END)
1317   - replacePatternInFile("MYSQL = False", "MYSQL = True", os.path.normpath("src/core/pyros_django/pyros/settings.py"))
1318   -
  1408 + replacePatternInFile("MYSQL = False", "MYSQL = True", os.path.normpath(
  1409 + "src/core/pyros_django/pyros/settings.py"))
  1410 +
1319 1411 #print(Colors.LOG_BLUE + "\r\n-----------------------------cd ..-----------------------------" + Colors.END)
1320 1412 # 2021 : no need to change folder in order to init the database (maybe the next line was forget to be commented because the previous line is commented)
1321   - #os.chdir("..")
  1413 + # os.chdir("..")
1322 1414  
1323 1415 # --------------------------------------------
1324 1416 # --- Executing migrations
1325 1417 # --------------------------------------------
1326   -
  1418 +
1327 1419 print(Colors.LOG_BLUE + "\r\n\r\n-----------------------------Migrate : executing pyros.py init_database-----------------------------" + Colors.END)
1328   - #TODO: from venv !!!
  1420 + # TODO: from venv !!!
1329 1421 try:
1330 1422 #os.system(GLOBAL_PYTHON+" pyros.py init_database")
1331 1423 res = os.system(GLOBAL_PYTHON+" pyros.py initdb")
... ... @@ -1345,46 +1437,37 @@ def install_database(venv):
1345 1437  
1346 1438 def notused_help():
1347 1439 print(
1348   - #"Welcome in the installation script of the pyros venv.\t\nPlease launch it from the install directory of pyros.\n\tIf you're on Ubuntu Debian or CentOS:\n\tlaunch it with sudo and <--prerequisites> or <-p> to install the prerequisites.\n\t-->sudo ./test_install.py -p\n\n\tFor the python packages launch it from the install directory of pyros without sudo and without parameter\n\t-->./test_install.py")
  1440 + # "Welcome in the installation script of the pyros venv.\t\nPlease launch it from the install directory of pyros.\n\tIf you're on Ubuntu Debian or CentOS:\n\tlaunch it with sudo and <--prerequisites> or <-p> to install the prerequisites.\n\t-->sudo ./test_install.py -p\n\n\tFor the python packages launch it from the install directory of pyros without sudo and without parameter\n\t-->./test_install.py")
1349 1441 "Welcome to the installation script of the pyros software.\t\n" + \
1350 1442 "Usage:\n" + \
1351 1443 " [python] ./pyros.py install [-p] [-d]"
1352 1444 )
1353 1445  
1354 1446  
1355   -
1356   -
1357   -
1358   -
1359   -
1360   -
1361   -
1362   -
1363   -
1364   -
1365 1447 """
1366 1448 ********************************************************************************
1367 1449 ********************************* main() FUNCTION ******************************
1368 1450 ********************************************************************************
1369 1451 """
  1452 +
  1453 +
1370 1454 def main():
1371 1455  
1372   - set_environment_variables_if_not_configured(ENV_PATH,ENV_SAMPLE_PATH)
  1456 + set_environment_variables_if_not_configured(ENV_PATH, ENV_SAMPLE_PATH)
1373 1457 pyros_launcher()
1374 1458  
1375   -
1376   -
  1459 +
1377 1460 # AVIRER
1378   -#@click.command()
1379   -#@click.argument('start')
  1461 +# @click.command()
  1462 +# @click.argument('start')
1380 1463 def oldmain():
1381 1464 '''
1382 1465 cli()
1383 1466 return
1384 1467 '''
1385   -
1386   - #fire.Fire(Commands)
1387   - #return
  1468 +
  1469 + # fire.Fire(Commands)
  1470 + # return
1388 1471  
1389 1472 # if len(sys.argv) == 3 and sys.argv[1].startswith("simulator"): SIMULATOR_CONFIG_FILE = sys.argv[2]
1390 1473 # print(sys.argv)
... ... @@ -1424,8 +1507,8 @@ def oldmain():
1424 1507 # sys.exit(pyros.exec())
1425 1508  
1426 1509  
1427   -
1428   -if __name__ == "__main__": main()
  1510 +if __name__ == "__main__":
  1511 + main()
1429 1512  
1430 1513  
1431 1514 """ INSTALLATION main function
... ...