Commit 34e7b6171870282f60cbad0dbba991f7cf8e8530

Authored by Alain Klotz
2 parents 4c0f2742 0a1f1d83
Exists in dev

Merge branch 'dev' of https://gitlab.irap.omp.eu/pyros-irap/pyros into dev

docker/Dockerfile
@@ -184,6 +184,7 @@ USER pyros_user @@ -184,6 +184,7 @@ USER pyros_user
184 184
185 RUN mkdir -p ./vendor/guitastro/install/ 185 RUN mkdir -p ./vendor/guitastro/install/
186 RUN mkdir -p ./install/ 186 RUN mkdir -p ./install/
  187 +RUN mkdir -p ./doc/doc_rst/
187 188
188 # Copy local host machine files to image 189 # Copy local host machine files to image
189 # Dockerfile contains this volume => ../..:/home/pyros_user/app 190 # Dockerfile contains this volume => ../..:/home/pyros_user/app
@@ -191,7 +192,7 @@ RUN mkdir -p ./install/ @@ -191,7 +192,7 @@ RUN mkdir -p ./install/
191 COPY ./vendor/guitastro/install/requirements.in ./vendor/guitastro/install/requirements.in 192 COPY ./vendor/guitastro/install/requirements.in ./vendor/guitastro/install/requirements.in
192 COPY ./install/requirements.txt ./install/requirements.txt 193 COPY ./install/requirements.txt ./install/requirements.txt
193 COPY ./install/requirements_dev.txt ./install/requirements_dev.txt 194 COPY ./install/requirements_dev.txt ./install/requirements_dev.txt
194 - 195 +COPY ./doc/doc_rst/requirements.txt ./doc/doc_rst/requirements.txt
195 # Adding local/bin to path to avoid pip warning 196 # Adding local/bin to path to avoid pip warning
196 ENV PATH "$PATH:/home/pyros_user/.local/bin" 197 ENV PATH "$PATH:/home/pyros_user/.local/bin"
197 198
@@ -219,3 +220,4 @@ RUN python3 -m pip install --user -r ./vendor/guitastro/install/requirements.txt @@ -219,3 +220,4 @@ RUN python3 -m pip install --user -r ./vendor/guitastro/install/requirements.txt
219 # Installing packages required for PyROS 220 # Installing packages required for PyROS
220 RUN python3 -m pip install --user -r ./install/requirements.txt 221 RUN python3 -m pip install --user -r ./install/requirements.txt
221 RUN python3 -m pip install --user -r ./install/requirements_dev.txt 222 RUN python3 -m pip install --user -r ./install/requirements_dev.txt
  223 +RUN python3 -m pip install --user -r ./doc/doc_rst/requirements.txt
docker/PYROS_DOCKER_UPDATE
1 #!/usr/bin/env bash 1 #!/usr/bin/env bash
2 2
3 # PRE-CONDITION : pyros container must be running 3 # PRE-CONDITION : pyros container must be running
4 - 4 +root_folder=$(pwd 2>&1)
5 # If no container is running Start it 5 # If no container is running Start it
6 if ! [ $(docker ps | grep 'pyros' | wc -l) -eq 4 ] 6 if ! [ $(docker ps | grep 'pyros' | wc -l) -eq 4 ]
7 then 7 then
@@ -12,11 +12,35 @@ fi @@ -12,11 +12,35 @@ fi
12 12
13 # 1) Update Guitastro 13 # 1) Update Guitastro
14 echo "Updating Guitastro source code" 14 echo "Updating Guitastro source code"
15 -(cd ../vendor/guitastro/ ; git pull) 15 +cd ../vendor/guitastro/
  16 +gitpull_output=$(git pull 2>&1)
  17 +echo $gitpull_output
  18 +check_files=("docker-compose.yml" "Dockerfile" "install/requirements.in")
  19 +for file in "${check_files[@]}"
  20 +do
  21 + if grep -q "$file" <<< "$gitpull_output"; then
  22 + echo "WARNING : One of docker related files has been changed. You should build again the PyROS Image"
  23 + return 1
  24 + fi
  25 +done
16 26
17 # 2) Update PyROS 27 # 2) Update PyROS
  28 +cd $root_folder
18 echo "Updating PyROS source code" 29 echo "Updating PyROS source code"
19 -git pull 30 +gitpull_output=$(git pull 2>&1)
  31 +echo $gitpull_output
  32 +check_files=("docker-compose.yml" "Dockerfile" "install/requirements.in")
  33 +for file in "${check_files[@]}"
  34 +do
  35 + if grep -q "$file" <<< "$gitpull_output"; then
  36 + echo "WARNING : One of docker related files has been changed. You should build again the PyROS Image"
  37 + return 1
  38 + fi
  39 +done
  40 +
  41 +# TODO : test if git pull worked and if git pull changed dockerfile, requirements.in, docker-compose.yml
  42 +# Tell user to rebuild
  43 +
20 44
21 # 3) Update all observatories with git repo 45 # 3) Update all observatories with git repo
22 for dir in ../../PYROS_OBSERVATORY/* ; do 46 for dir in ../../PYROS_OBSERVATORY/* ; do
@@ -84,7 +84,8 @@ AGENTS = { @@ -84,7 +84,8 @@ AGENTS = {
84 "A_Basic": "majordome/agent", 84 "A_Basic": "majordome/agent",
85 "A_Triton": "majordome/agent", 85 "A_Triton": "majordome/agent",
86 "A_MCO":"majordome/agent", 86 "A_MCO":"majordome/agent",
87 - "A_Majordome":"majordome/agent" 87 + "A_Majordome":"majordome/agent",
  88 + "AD_PLC":"env_monitor"
88 } 89 }
89 # AGENTS = ["agentX", "webserver", "monitoring", "majordome", "scheduler", "alert_manager"] 90 # AGENTS = ["agentX", "webserver", "monitoring", "majordome", "scheduler", "alert_manager"]
90 # AGENTS = ["all", "webserver", "monitoring", "majordome", "scheduler", "alert"] 91 # AGENTS = ["all", "webserver", "monitoring", "majordome", "scheduler", "alert"]
@@ -507,7 +508,7 @@ def shell(): @@ -507,7 +508,7 @@ def shell():
507 508
508 # execProcess("python install.py install") 509 # execProcess("python install.py install")
509 if not test_mode(): 510 if not test_mode():
510 - execProcessFromVenv("manage.py shell",foreground=True) 511 + execProcess("./manage.py shell",foreground=True)
511 # Go back to the initial dir 512 # Go back to the initial dir
512 # os.chdir("../") 513 # os.chdir("../")
513 change_dir("PREVIOUS") 514 change_dir("PREVIOUS")
@@ -1009,8 +1010,8 @@ def start(agent: str, configfile: str, observatory: str, unit: str, computer_hos @@ -1009,8 +1010,8 @@ def start(agent: str, configfile: str, observatory: str, unit: str, computer_hos
1009 current_dir = os.getcwd() 1010 current_dir = os.getcwd()
1010 1011
1011 # OLD format agents: majordome, monitoring, alert... 1012 # OLD format agents: majordome, monitoring, alert...
1012 - cmd = "start_agent.py " + agent_name + " " + configfile  
1013 - 1013 + #cmd = "start_agent.py " + agent_name + " " + configfile
  1014 + cmd = ""
1014 # Agent "webserver" 1015 # Agent "webserver"
1015 if agent_name == "webserver": 1016 if agent_name == "webserver":
1016 cmd = "manage.py runserver 127.0.0.1:8080" 1017 cmd = "manage.py runserver 127.0.0.1:8080"
@@ -1022,7 +1023,7 @@ def start(agent: str, configfile: str, observatory: str, unit: str, computer_hos @@ -1022,7 +1023,7 @@ def start(agent: str, configfile: str, observatory: str, unit: str, computer_hos
1022 # if not test_mode(): execProcessFromVenv("start_agent.py " + agent_name + " " + configfile) 1023 # if not test_mode(): execProcessFromVenv("start_agent.py " + agent_name + " " + configfile)
1023 1024
1024 # Agent "agentM", "agentA", "agentB", "agentX", ... 1025 # Agent "agentM", "agentA", "agentB", "agentX", ...
1025 - elif agent_name.startswith("A_") or agent_name.startswith("A_"): 1026 + elif agent_name.startswith("A_") or agent_name.startswith("AD_"):
1026 # Run agent without actual commands sent to devices (FOR_REAL=False) 1027 # Run agent without actual commands sent to devices (FOR_REAL=False)
1027 # agentX.run(FOR_REAL=True) 1028 # agentX.run(FOR_REAL=True)
1028 ''' 1029 '''
@@ -1299,10 +1300,11 @@ def _update_plantuml_diags(): @@ -1299,10 +1300,11 @@ def _update_plantuml_diags():
1299 def _update_api_doc(): 1300 def _update_api_doc():
1300 print(os.getcwd()) 1301 print(os.getcwd())
1301 DOC_RST_PATH = "doc/doc_rst/" 1302 DOC_RST_PATH = "doc/doc_rst/"
1302 - # 0) Upgrade pip if new version available  
1303 - _venv_pip_install2('pip', '--upgrade')  
1304 - # 1) install/update Sphinx requirements (only if not yet installed)  
1305 - _venv_pip_install2(DOC_RST_PATH+'requirements.txt', '-r') 1303 + if not WITH_DOCKER:
  1304 + # 0) Upgrade pip if new version available
  1305 + _venv_pip_install2('pip', '--upgrade')
  1306 + # 1) install/update Sphinx requirements (only if not yet installed)
  1307 + _venv_pip_install2(DOC_RST_PATH+'requirements.txt', '-r')
1306 # 2) make html doc from RST doc 1308 # 2) make html doc from RST doc
1307 # cd doc/sourcedoc/ 1309 # cd doc/sourcedoc/
1308 change_dir(DOC_RST_PATH) 1310 change_dir(DOC_RST_PATH)
@@ -1360,6 +1362,7 @@ def change_dir(path): @@ -1360,6 +1362,7 @@ def change_dir(path):
1360 def _check_agent(agent): 1362 def _check_agent(agent):
1361 # Check that agent exists 1363 # Check that agent exists
1362 if agent != "all" and agent not in AGENTS.keys(): 1364 if agent != "all" and agent not in AGENTS.keys():
  1365 +
1363 # Check if multiple agents: 1366 # Check if multiple agents:
1364 print("This agent does not exist") 1367 print("This agent does not exist")
1365 print("Here is the allowed list of agents:") 1368 print("Here is the allowed list of agents:")
src/core/pyros_django/api/serializers.py
@@ -54,7 +54,7 @@ class SPPeriodSerializer(serializers.ModelSerializer): @@ -54,7 +54,7 @@ class SPPeriodSerializer(serializers.ModelSerializer):
54 scientific_program = ScientificProgramSerializer(read_only=True) 54 scientific_program = ScientificProgramSerializer(read_only=True)
55 class Meta: 55 class Meta:
56 model = SP_Period 56 model = SP_Period
57 - fields = ["id", "scientific_program","period", "quota_remaining", "over_quota_duration_remaining", "token_remaining"] 57 + fields = ["id", "scientific_program","period"]
58 58
59 class SequenceSerializer(serializers.ModelSerializer): 59 class SequenceSerializer(serializers.ModelSerializer):
60 pyros_user = UserSerializer(read_only=True) 60 pyros_user = UserSerializer(read_only=True)
src/core/pyros_django/dashboard/templates/dashboard/observation_index.html
@@ -114,9 +114,10 @@ @@ -114,9 +114,10 @@
114 </div> 114 </div>
115 </li> 115 </li>
116 {% endif %} 116 {% endif %}
  117 + {% comment %}
117 <li> 118 <li>
118 <div class="all-info"> 119 <div class="all-info">
119 - <a href="{% url "alerts_list" %}"> 120 + <a href="{% url "alerts_list" %}">
120 <h3>Alerts</h3> 121 <h3>Alerts</h3>
121 <img src="{% static "media/alert.png" %}" alt="html5" /> 122 <img src="{% static "media/alert.png" %}" alt="html5" />
122 </a> 123 </a>
@@ -126,10 +127,11 @@ @@ -126,10 +127,11 @@
126 <div class="all-info"> 127 <div class="all-info">
127 <a href="{% url "current_schedule" %}"> 128 <a href="{% url "current_schedule" %}">
128 <h3>Schedule</h3> 129 <h3>Schedule</h3>
129 - <img src="{% static "media/schedule.png" %}" alt="html5" /> 130 + <img src="{% static "media/schedule.png" %}" alt="html5" />
130 </a> 131 </a>
131 </div> 132 </div>
132 </li> 133 </li>
  134 + {% endcomment %}
133 {# TBD #} 135 {# TBD #}
134 {% if USER_LEVEL|ifinlist:"Admin,Operator,Unit-PI,Management board member,Observer,TAC" %} 136 {% if USER_LEVEL|ifinlist:"Admin,Operator,Unit-PI,Management board member,Observer,TAC" %}
135 <li> 137 <li>
src/core/pyros_django/env_monitor/AD_PLC.py
@@ -10,7 +10,7 @@ from src.core.pyros_django.majordome.agent.Agent import Agent, build_agent, log, @@ -10,7 +10,7 @@ from src.core.pyros_django.majordome.agent.Agent import Agent, build_agent, log,
10 from src.core.pyros_django.devices.PLC import PLCController 10 from src.core.pyros_django.devices.PLC import PLCController
11 from src.core.pyros_django.env_monitor.plc_checker import PlcChecker 11 from src.core.pyros_django.env_monitor.plc_checker import PlcChecker
12 from common.models import * 12 from common.models import *
13 -from env_monitor.models import Sensors_data_last_value 13 +from env_monitor.models import Sensors_data_last_value, Sensors_data
14 from dashboard.config_pyros import ConfigPyros 14 from dashboard.config_pyros import ConfigPyros
15 15
16 16
src/core/pyros_django/env_monitor/plc_checker.py
@@ -149,11 +149,12 @@ class PlcChecker(object): @@ -149,11 +149,12 @@ class PlcChecker(object):
149 # ok with ./start_agent agentM 149 # ok with ./start_agent agentM
150 #plc_config_dir = "./src/core/pyros_django/monitoring/" 150 #plc_config_dir = "./src/core/pyros_django/monitoring/"
151 # ok with ./pyros start agentM 151 # ok with ./pyros start agentM
152 - plc_config_dir = Path("../env_monitor/") 152 + #plc_config_dir = Path("../env_monitor/")
  153 + plc_config_dir = os.path.join(os.environ.get("DJANGO_PATH"), "env_monitor")
153 # When executing test, we're not in the monitoring folder 154 # When executing test, we're not in the monitoring folder
154 if("pyros_django" == os.getcwd().split("/")[-1]): 155 if("pyros_django" == os.getcwd().split("/")[-1]):
155 plc_config_dir = Path("./env_monitor/") 156 plc_config_dir = Path("./env_monitor/")
156 - _file = open(plc_config_dir/"plc_config.json" , 'w') 157 + _file = open(os.path.join(plc_config_dir,"plc_config.json") , 'w')
157 _file.write(json.dumps(_struct)) 158 _file.write(json.dumps(_struct))
158 _file.close() 159 _file.close()
159 logger.info("Saved : plc_config.json") 160 logger.info("Saved : plc_config.json")
src/core/pyros_django/misc/fixtures/initial_fixture_dev_TZ.json
@@ -234,7 +234,14 @@ @@ -234,7 +234,14 @@
234 "user": 4 234 "user": 4
235 } 235 }
236 }, 236 },
237 - 237 + {
  238 + "model": "user_mgmt.SP_Period_User",
  239 + "pk": 4,
  240 + "fields": {
  241 + "SP_Period": 1,
  242 + "user": 17
  243 + }
  244 + },
238 { 245 {
239 "model": "devices.telescope", 246 "model": "devices.telescope",
240 "pk": 1, 247 "pk": 1,
@@ -358,7 +365,7 @@ @@ -358,7 +365,7 @@
358 "pk": 2, 365 "pk": 2,
359 "fields": { 366 "fields": {
360 "name": "CNRS", 367 "name": "CNRS",
361 - "f_quota": 0.2 368 + "quota_f": 0.2
362 } 369 }
363 }, 370 },
364 { 371 {
@@ -366,7 +373,7 @@ @@ -366,7 +373,7 @@
366 "pk": 1, 373 "pk": 1,
367 "fields": { 374 "fields": {
368 "name": "CNES", 375 "name": "CNES",
369 - "f_quota": 0.8 376 + "quota_f": 0.8
370 } 377 }
371 }, 378 },
372 { 379 {
@@ -493,6 +500,37 @@ @@ -493,6 +500,37 @@
493 }, 500 },
494 { 501 {
495 "model": "user_mgmt.pyrosuser", 502 "model": "user_mgmt.pyrosuser",
  503 + "pk": 17,
  504 + "fields": {
  505 + "password": "pbkdf2_sha256$100000$LdWOwxlQqzYU$n5YIpgps3de5U8bAx3sVmhFhwXCHgczKszJG8SuVyjs=",
  506 + "last_login": null,
  507 + "is_superuser": false,
  508 + "first_name": "Manuel",
  509 + "last_name": "Pavy",
  510 + "email": "Manuel.Pavy@cnes.fr",
  511 + "is_staff": false,
  512 + "date_joined": "2021-05-20T09:03:37.108Z",
  513 + "username": "Manuel.Pavy@cnes.fr",
  514 + "is_active": true,
  515 + "first_time": false,
  516 + "country": 1,
  517 + "desc": null,
  518 + "created": "2021-05-20T09:03:37.108Z",
  519 + "updated": "2021-05-20T09:03:37.138Z",
  520 + "tel": null,
  521 + "address": "18 Avenue Edouard Belin, 31400 Toulouse",
  522 + "laboratory": "CNES",
  523 + "institute": 1,
  524 + "validator": 1,
  525 + "groups": [],
  526 + "user_permissions": [],
  527 + "user_level": [
  528 + 2
  529 + ]
  530 + }
  531 + },
  532 + {
  533 + "model": "user_mgmt.pyrosuser",
496 "pk": 5, 534 "pk": 5,
497 "fields": { 535 "fields": {
498 "password": "pbkdf2_sha256$100000$LdWOwxlQqzYU$n5YIpgps3de5U8bAx3sVmhFhwXCHgczKszJG8SuVyjs=", 536 "password": "pbkdf2_sha256$100000$LdWOwxlQqzYU$n5YIpgps3de5U8bAx3sVmhFhwXCHgczKszJG8SuVyjs=",
src/core/pyros_django/user_mgmt/models.py
@@ -475,7 +475,7 @@ class ScientificProgram(models.Model): @@ -475,7 +475,7 @@ class ScientificProgram(models.Model):
475 science_theme = models.ForeignKey(ScienceTheme, on_delete=models.DO_NOTHING, related_name="scientific_program_theme", default=1) 475 science_theme = models.ForeignKey(ScienceTheme, on_delete=models.DO_NOTHING, related_name="scientific_program_theme", default=1)
476 is_auto_validated = models.BooleanField(default=False) 476 is_auto_validated = models.BooleanField(default=False)
477 objects = ScientificProgramManager() 477 objects = ScientificProgramManager()
478 - quota = models.ForeignKey(Quota, on_delete=models.DO_NOTHING,related_name="scientific_program_quotas", blank=True, null=True) 478 + quota = models.ForeignKey(Quota, on_delete=models.DO_NOTHING, related_name="scientific_program_quotas", blank=True, null=True)
479 quota_f = models.FloatField( 479 quota_f = models.FloatField(
480 validators=[MinValueValidator(0), MaxValueValidator(1)], blank=True, null=True) 480 validators=[MinValueValidator(0), MaxValueValidator(1)], blank=True, null=True)
481 481