PYROS-IRAP / pyros

Commit 76a6bf8be1f4c9e0812b12b8e7ba68ea8ee692e5

Authored by Etienne Pallier
2 parents 02d535ce 53b75e7b
Exists in dev

Merge branch 'dev' of https://gitlab.irap.omp.eu/epallier/pyros into dev

Showing 15 changed files with 322 additions and 252 deletions   Show diff stats
Inline Side-by-side
CHANGELOG
  Diff comments   View file @76a6bf8
  1 +04-04-2023 (AKo): v0.6.21.0
  2 + - Improve check of ObsConfig load function and renaming functions of ObsConfig & PyrosUser
  3 + - Add schema (Md file) to explain workflow when user import a sequence file
  4 + - Enable real email when in prod (settings.py)
  5 +
1 6 20-03-2023 (AKo): v0.6.20.0
2 7 - Add night_id to sequences, create folder for night_id
3 8 - Change sequences_pickle location (relocate in data folder)
... ...
VERSION
  Diff comments   View file @76a6bf8
1   -0.6.17.0
2 1 \ No newline at end of file
  2 +0.6.21.0
3 3 \ No newline at end of file
... ...
install/requirements.txt
  Diff comments   View file @76a6bf8
... ... @@ -6,7 +6,7 @@ django-admin-tools==0.9.1
6 6 django-bootstrap3==21.1
7 7 django-extensions==3.1.5
8 8 # for Django Rest Framework (DRF)
9   -djangorestframework=3.14.0
  9 +djangorestframework==3.14.0
10 10 # for Choices
11 11 django-model-utils==4.2.0
12 12 django-suit==0.2.28
... ...
src/core/pyros_django/alert_manager/tests.py
  Diff comments   View file @76a6bf8
... ... @@ -8,8 +8,8 @@ from django.conf import settings
8 8  
9 9 from common.models import *
10 10  
11   -from utils.Logger import *
12   -log = setupLogger("test", "test")
  11 +#from utils.Logger import *
  12 +#log = setupLogger("test", "test")
13 13  
14 14 TEST_FILE = "unittest_voevent.xml"
15 15  
... ...
src/core/pyros_django/common/tests.py
  Diff comments   View file @76a6bf8
1 1 # Santdard imports
2   -from utils.Logger import *
  2 +#from utils.Logger import *
3 3  
4 4 # Django imports
5 5 from django.test import TestCase
... ... @@ -11,7 +11,7 @@ from user_manager.models import Country
11 11 from common.models import *
12 12 from common.RequestBuilder import RequestBuilder
13 13  
14   -log = setupLogger("common", "common")
  14 +#log = setupLogger("common", "common")
15 15  
16 16 """
17 17 class RequestBuilderTests(TestCase):
... ...
src/core/pyros_django/dashboard/views.py
  Diff comments   View file @76a6bf8
... ... @@ -4,7 +4,7 @@ import re
4 4 import sys
5 5 import datetime
6 6 from datetime import timezone
7   -import utils.Logger as l
  7 +#import utils.Logger as l
8 8 import json
9 9 from random import randint
10 10 import time,os
... ... @@ -54,7 +54,7 @@ import vendor.guitastro.src.guitastro as guitastro
54 54 SUN_ELEV_DAY_THRESHOLD = -10
55 55 MAX_LOGS_LINES = 100
56 56  
57   -log = l.setupLogger("dashboard", "dashboard")
  57 +#log = l.setupLogger("dashboard", "dashboard")
58 58  
59 59 def index(request):
60 60 config = OBSConfig(os.environ["PATH_TO_OBSCONF_FILE"],os.environ["unit_name"])
... ...
src/core/pyros_django/devices/Device.py
  Diff comments   View file @76a6bf8
... ... @@ -4,7 +4,7 @@ import configparser
4 4 import abc
5 5 from pathlib import Path
6 6 import select
7   -import utils.Logger as L
  7 +#import utils.Logger as L
8 8 import time
9 9 import os
10 10 from common.models import Log
... ... @@ -19,7 +19,7 @@ RECONNECT_TIMEOUT_SECONDS = 20
19 19  
20 20 class DeviceController():
21 21 __metaclass__ = abc.ABCMeta
22   - logger = L.setupLogger("DeviceLogger", "Devices")
  22 + #logger = L.setupLogger("DeviceLogger", "Devices")
23 23 name = ""
24 24 #config_file = "../config/socket_config.ini"
25 25 my_parent_abs_dir = Path(__file__).resolve().parent
... ... @@ -170,7 +170,8 @@ class DeviceController():
170 170 def log(self, device_name: str, message: str):
171 171 Log.objects.create(agent=self.name, message=message)
172 172 if DEBUG_FILE and settings.DEBUG:
173   - self.logger.info("From device : " + device_name + " -> " + message)
  173 + #self.logger.info("From device : " + device_name + " -> " + message)
  174 + pass
174 175 return (0)
175 176  
176 177 def connect(self):
... ...
src/core/pyros_django/obsconfig/obsconfig_class.py
  Diff comments   View file @76a6bf8
... ... @@ -59,15 +59,17 @@ class OBSConfig:
59 59 self.CONFIG_PATH = os.path.dirname(observatory_config_file)+"/"
60 60 self.obs_config_path = self.CONFIG_PATH
61 61 #self.CONFIG_PATH = self.obs_config_path
  62 + # If the pickle doesn't exists
62 63 if os.path.isfile(self.CONFIG_PATH+self.pickle_file) == False:
63 64 return True
64 65 else:
  66 + # pickle file exists
65 67 pickle_file_mtime = os.path.getmtime(
66 68 self.CONFIG_PATH+self.pickle_file)
67 69 obs_config_mtime = os.path.getmtime(observatory_config_file)
68 70  
69   - obs_config = self.read_and_check_config_file(
70   - observatory_config_file)
  71 + #obs_config = self.read_and_check_config_file(
  72 + # observatory_config_file)
71 73 if obs_config_mtime > pickle_file_mtime:
72 74 # create obs file (yaml) from pickle["obsconfig"] with date of pickle within history folder-> nom ficher + année + mois + jour + datetime (avec secondes) -> YYYY/MM/DD H:m:s
73 75 pickle_datetime = datetime.utcfromtimestamp(
... ... @@ -81,23 +83,32 @@ class OBSConfig:
81 83 with open(file_name, 'w') as f:
82 84 f.write(config_file.read())
83 85 return True
84   - if obs_config == None:
85   - print(
86   - f"Error when trying to read config file (path of config file : {observatory_config_file}")
87   - return -1
88   - self.obs_config = obs_config
89   - # check last date of modification for devices files
90   - for device in self.obs_config["OBSERVATORY"]["INVENTORY"]["DEVICES"]:
91   - device_file = self.CONFIG_PATH+device["DEVICE"]["file"]
92   - device_file_mtime = os.path.getmtime(device_file)
93   - if device_file_mtime > pickle_file_mtime:
94   - return True
95   -
96   - for computer in self.obs_config["OBSERVATORY"]["INVENTORY"]["COMPUTERS"]:
97   - computer_file = self.CONFIG_PATH+computer["COMPUTER"]["file"]
98   - computer_file_mtime = os.path.getmtime(computer_file)
99   - if computer_file_mtime > pickle_file_mtime:
100   - return True
  86 + # if obs_config == None:
  87 + # print(
  88 + # f"Error when trying to read config file (path of config file : {observatory_config_file}")
  89 + # return -1
  90 +
  91 + for file in os.listdir(self.CONFIG_PATH):
  92 + # loop on each file in obsconfig folder exluding pickle itself and global obsconfig file (observatory_config_file)
  93 + if file != observatory_config_file and ".p" not in file:
  94 + file_path = self.CONFIG_PATH + file
  95 + if os.path.getmtime(file_path) > pickle_file_mtime:
  96 + return True
  97 +
  98 + # Old version not optimized
  99 + # self.obs_config = obs_config
  100 + # # check last date of modification for devices files
  101 + # for device in self.obs_config["OBSERVATORY"]["INVENTORY"]["DEVICES"]:
  102 + # device_file = self.CONFIG_PATH+device["DEVICE"]["file"]
  103 + # device_file_mtime = os.path.getmtime(device_file)
  104 + # if device_file_mtime > pickle_file_mtime:
  105 + # return True
  106 +
  107 + # for computer in self.obs_config["OBSERVATORY"]["INVENTORY"]["COMPUTERS"]:
  108 + # computer_file = self.CONFIG_PATH+computer["COMPUTER"]["file"]
  109 + # computer_file_mtime = os.path.getmtime(computer_file)
  110 + # if computer_file_mtime > pickle_file_mtime:
  111 + # return True
101 112 return False
102 113  
103 114 def load(self, observatory_config_file):
... ... @@ -1119,7 +1130,7 @@ class OBSConfig:
1119 1130 uneditable_fields[attribute] = attributes[attribute]
1120 1131 return uneditable_fields
1121 1132  
1122   - def getEditableAttributesOfChannel(self, unit_name: str, channel_name: str) -> list:
  1133 + def getEditableChannelAttributes(self, unit_name: str, channel_name: str) -> list:
1123 1134 capabilities = self.getChannelCapabilities(unit_name, channel_name)
1124 1135 # merged_result = {}
1125 1136 # for capability in capabilities:
... ... @@ -1154,7 +1165,7 @@ class OBSConfig:
1154 1165 def getAlbumByName(self, unit_name: str, name_of_album):
1155 1166 return self.get_albums(unit_name)["albums"][name_of_album]
1156 1167  
1157   - def getEditableAttributesOfMount(self, unit_name):
  1168 + def getEditableMountAttributes(self, unit_name):
1158 1169 capabilities = self.get_device_capabilities(
1159 1170 self.get_device_for_agent(unit_name, "mount")["name"])
1160 1171 merged_result = []
... ... @@ -1347,7 +1358,7 @@ def main():
1347 1358 # print(config.getChannelCapabilities(unit_name,"OpticalChannel_up"))
1348 1359 # print(config.get_channel_groups(unit_name))
1349 1360 # print(config.getEditableAttributesOfCapability(config.getChannelCapabilities(unit_name,"OpticalChannel_up")[0]))
1350   - # print(config.getEditableAttributesOfChannel(unit_name,"OpticalChannel_up"))
  1361 + # print(config.getEditableChannelAttributes(unit_name,"OpticalChannel_up"))
1351 1362 config = OBSConfig("../../../../privatedev/config/tnc/observatory_tnc.yml")
1352 1363 unit_name = config.get_units_name()[0]
1353 1364 # dc = config.getDeviceControllerNameForAgent(unit_name,"mount")[0]
... ... @@ -1356,8 +1367,8 @@ def main():
1356 1367 # print(config.getChannelCapabilities(unit_name,"OpticalChannel_down2"))
1357 1368 # print(config.get_channel_groups(unit_name))
1358 1369 # print(config.getEditableAttributesOfCapability(config.getChannelCapabilities(unit_name,"OpticalChannel_down2")[0]))
1359   - # print(config.getEditableAttributesOfChannel(unit_name,"OpticalChannel_down2"))
1360   - #print(config.getEditableAttributesOfMount(unit_name))
  1370 + # print(config.getEditableChannelAttributesnit_name,"OpticalChannel_down2"))
  1371 + #print(config.getEditableMountAttributes(unit_name))
1361 1372 config.get_devices()
1362 1373  
1363 1374 #print(config.get_output_data_device("TAROT_meteo").get("CV7"))
... ...
src/core/pyros_django/pyros/settings.py
  Diff comments   View file @76a6bf8
... ... @@ -231,9 +231,14 @@ INSTALLED_APPS = [
231 231 'obsconfig',
232 232 'dashboard',
233 233 "api",
  234 + #"silk",
234 235 #'kombu.transport.django'
235 236 ]
236 237  
  238 +#silk
  239 +
  240 +#SILKY_PYTHON_PROFILER = True
  241 +
237 242 CHANNEL_LAYERS = {
238 243 "default": {
239 244  
... ... @@ -263,7 +268,8 @@ MIDDLEWARE = [
263 268 'django.middleware.clickjacking.XFrameOptionsMiddleware',
264 269 # For debug_toolbar
265 270 'debug_toolbar.middleware.DebugToolbarMiddleware',
266   -
  271 + # silk
  272 + #"silk.middleware.SilkyMiddleware",
267 273 ]
268 274  
269 275 ROOT_URLCONF = 'pyros.urls'
... ... @@ -333,7 +339,7 @@ if MYSQL:
333 339 #CONN_MAX_AGE = 0
334 340 # - Always persistant
335 341 #CONN_MAX_AGE = None
336   - 'CONN_MAX_AGE': 500,
  342 + 'CONN_MAX_AGE': 10,
337 343 '''
338 344 (See https://docs.djangoproject.com/fr/2.1/topics/testing/overview/#the-test-database)
339 345 Optional, but this allows to remember the default django test database name
... ... @@ -493,7 +499,12 @@ else:
493 499 # from django.core.cache import cache
494 500 # cache.clear()
495 501  
496   -EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
  502 +if os.environ.get("IS_PROD") and os.environ.get("IS_PROD") == True :
  503 + # If in prod, use real email
  504 + EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
  505 +else:
  506 + # else email aren't sent and are print in console instead
  507 + EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
497 508  
498 509 python_version = subprocess.run( "python --version | cut -d ' ' -f 2 | cut -d '.' -f 1,2",shell=True,stdout=subprocess.PIPE,universal_newlines=True)
499 510 python_version = python_version.stdout
... ...
src/core/pyros_django/routine_manager/functions.py
  Diff comments   View file @76a6bf8
... ... @@ -12,155 +12,33 @@ from django.db import IntegrityError
12 12 # Project imports
13 13 from src.core.pyros_django.obsconfig.obsconfig_class import OBSConfig
14 14 from django.http import HttpRequest
  15 +#from silk.profiling.profiler import silk_profile
15 16  
  17 +#@silk_profile(name="check_sequence_file")
16 18 def check_sequence_file_validity_and_save(yaml_content: dict, request: HttpRequest):
17 19 ''' Create a sequence in DB from the uploaded sequence (yaml_content) '''
18 20  
19   - # Get boolean to simplified to know if the file is written in simplified mode (i.e. : Each field of the form is directly associated to its value)
20   - is_simplified = yaml_content.get("simplified", False)
21   - # Get scientific programs for the user who is submitting the sequence file
22   - user_sp = request.user.get_scientific_programs()
23   - # From user sp, get all SP that can observe / submit sequence for the current period
24   - sp_list = ScientificProgram.objects.observable_programs().filter(id__in=user_sp)
25   -
26 21 # Create a sequence seq object (from yaml_content) to be saved in DB
27 22 seq = Sequence.objects.create()
28 23 seq.pyros_user = PyrosUser.objects.get(id=request.user.id)
29 24  
30 25 # Get the unit config
31 26 unit_name = os.environ["unit_name"]
  27 + #with silk_profile(name="init obsconfig"):
32 28 config = OBSConfig(os.environ["PATH_TO_OBSCONF_FILE"], unit_name)
33   -
34   - # Create a Sequence form
35   - sequence_form = SequenceForm(instance=seq, data_from_config=config.getEditableMountAttributes(config.unit_name), layouts = config.get_layouts(config.unit_name), sp_list=sp_list)
36 29 result = {
37 30 "succeed": True,
38 31 "errors": [],
39 32 }
40   -
41   - if is_simplified:
42   - seq.scientific_program = sp_list[yaml_content["sequence"]["scientific_program"]]
  33 + # Get boolean to simplified to know if the file is written in simplified mode (i.e. : Each field of the form is directly associated to its value)
  34 + is_simplified = yaml_content.get("simplified", False)
  35 + # Get scientific programs for the user who is submitting the sequence file
  36 + user_sp = request.user.get_scientific_programs()
  37 + process_sequence(yaml_content, seq, config, is_simplified, result, user_sp)
43 38  
44   - else:
45   - # pour la lisibilité du code (et éviter la redondance)
46   - # get scientific program field's attributes
47   - yaml_seq_sp = yaml_content["sequence"]["scientific_program"]
48   - sp_index_value = yaml_seq_sp["value"]
49   - values = yaml_seq_sp["values"]
50   - # Check if index of sp is valid (in range of possible index from values)
51   - if sp_index_value < 0 or sp_index_value > len(values):
52   - result["errors"].append(f"SP value isn't valid, index out of bounds ({sp_index_value} > {len(values)})")
53   - sp_index_value = 0
54   - chosen_sp = ScientificProgram.objects.get(name=values[sp_index_value])
55   - # If the sp is associated to that user, associate the sp to the sequence
56   - if chosen_sp in sp_list:
57   - #seq.scientific_program = ScientificProgram.objects.get(name=yaml_content["sequence"]["scientific_program"]["values"][sp_index_value])
58   - seq.scientific_program = chosen_sp
59   - else:
60   - result["errors"].append(f"SP {chosen_sp.name} is not assigned to that user ")
  39 + process_albums(yaml_content, result, config, seq, is_simplified)
61 40  
62   - seq.config_attributes = {}
63   -
64   - # Fill all Sequence form fields
65   - # keys() inutile ? => for field in sequence_form.fields :
66   - # Sinon, y a aussi => for key,val in sequence_form.fields.items():
67   - # => Ca éviterait de faire => sequence_form.fields[field] pour recuperer "val"
68   - for field, field_attribute in sequence_form.fields.items():
69 41  
70   - #if sequence_form.fields[field].required == False or field == "scientific_program":
71   - if not field_attribute.required or field=="scientific_program":
72   - continue
73   - # pour lisibilité, simplicité et éviter redondance
74   - yaml_field = yaml_content["sequence"][field]
75   - value = yaml_field if is_simplified else yaml_field["value"]
76   - ''' (orig)
77   - if is_simplified:
78   - value = yaml_content["sequence"][field]
79   - else:
80   - value = yaml_content["sequence"][field]["value"]
81   - '''
82   - # If the current field of the sequence isn't found in the file, add an error message to inform the user the missing field
83   - if field not in yaml_content["sequence"]:
84   - result["errors"].append(f"{field} not in yaml file")
85   - else:
86   - if is_simplified:
87   - # If the field is a choicefield, get choices and associate the index to the real value
88   - if sequence_form.fields[field].__dict__.get("_choices"):
89   - # y a pas conflit ici avec la variable "value" définie au-dessus ? -> Non car on transforme l'ancien value qui est un index en une vraie valeur
90   - values = [value[0] for value in sequence_form.fields[field].__dict__.get("_choices")]
91   - value = values[value]
92   - else:
93   - if yaml_field.get("values"):
94   - # Transform the original value which is an index to a "real" value from the "values" attributes
95   - index_value = yaml_field["value"]
96   - values = yaml_field["values"]
97   - if index_value < 0 or index_value > len(yaml_field["values"]):
98   - result["errors"].append(f"Value of {field} isn't valid, index out of bounds ({index_value} > {len(values)})")
99   - index_value = 0
100   - value = yaml_field["values"][index_value]
101   - else:
102   - # Transform the string value to a datetime value
103   - if field == "start_date":
104   - if type(value) != datetime.datetime:
105   - #value = datetime.datetime.strptime(yaml_content["sequence"][field]["value"],'%d/%m/%Y %H:%M:%S')
106   - # ISO format
107   - value = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%f')
108   - seq.__dict__[field] = value
109   - ''' (orig)
110   - else:
111   - if not is_simplified:
112   - if yaml_content["sequence"][field].get("values"):
113   - index_value = yaml_content["sequence"][field]["value"]
114   - values = yaml_content["sequence"][field]["values"]
115   - if index_value < 0 or index_value > len(yaml_content["sequence"][field]["values"]):
116   - result["errors"].append(f"Value of {field} isn't valid, index out of bounds ({index_value} > {len(values)})")
117   - index_value = 0
118   - value = yaml_content["sequence"][field]["values"][index_value]
119   - else:
120   - if field == "start_date":
121   - if type(value) != datetime.datetime:
122   - #value = datetime.datetime.strptime(yaml_content["sequence"][field]["value"],'%d/%m/%Y %H:%M:%S')
123   - # ISO format
124   - value = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%f')
125   - seq.__dict__[field] = value
126   - else:
127   - if sequence_form.fields[field].__dict__.get("_choices"):
128   - values = [value[0] for value in sequence_form.fields[field].__dict__.get("_choices")]
129   - value = values[value]
130   - '''
131   -
132   - # suffisant ? => if field in seq.__dict__
133   - # If field is an attribute of the sequence, associate the field to the value
134   - if field in seq.__dict__:
135   - seq.__dict__[field] = value
136   - else:
137   - # else associate field & value in config_attributes sequence's field (JsonField) = variable fields of an sequence
138   - seq.config_attributes[field] = value
139   -
140   - # Create ALBUMS
141   - albums_from_file = yaml_content["sequence"]["ALBUMS"]
142   - chosen_layout = seq.config_attributes["layout"]
143   - if type(chosen_layout) == int:
144   - layouts = config.get_layouts(config.unit_name)["layouts"]
145   - chosen_layout = list(layouts.keys())[chosen_layout]
146   - # Get album of the selected layout
147   - layout_albums = config.getLayoutByName(unit_name=config.unit_name, name_of_layout=chosen_layout)["ALBUMS"]
148   -
149   - # check if we have all the albums of that layout described in the sequence file
150   - if len(layout_albums) == len(albums_from_file):
151   - for album in albums_from_file:
152   - album = album["Album"]
153   - if album["name"] not in layout_albums:
154   - result["errors"].append(f"Album {album['name']} is not the chosen layout")
155   - else:
156   - # Create album
157   - Album.objects.create(name=album["name"], sequence=seq, complete=True)
158   - # Create plan for that album
159   - plans = [a["Album"].get("Plans") for a in albums_from_file]
160   - process_plans(plans, result, is_simplified, config, album, seq)
161   - else:
162   - result["errors"].append(f"The number of albums doesn't correspond to the chosen layout")
163   -
164 42 # optim possible ?
165 43 #[ process_plans(a["Album"].get("Plans")) for a in albums_from_file ]
166 44 # Puis écrire la fonction process_plans()
... ... @@ -291,19 +169,51 @@ def check_sequence_file_validity_and_save(yaml_content: dict, request: HttpReque
291 169 result["sequence_id"] = seq.id
292 170 return result
293 171  
  172 +def process_albums(yaml_content, result, config, seq, is_simplified):
  173 + # Create ALBUMS
  174 + albums_from_file = yaml_content["sequence"]["ALBUMS"]
  175 + chosen_layout = seq.config_attributes["layout"]
  176 + if type(chosen_layout) == int:
  177 + #with silk_profile(name="Get layout from config"):
  178 + layouts = config.get_layouts(config.unit_name)["layouts"]
  179 + chosen_layout = list(layouts)[chosen_layout]
  180 + # Get album of the selected layout
  181 + #with silk_profile(name="Get album of layout from config"):
  182 + layout_albums = config.getLayoutByName(unit_name=config.unit_name, name_of_layout=chosen_layout)["ALBUMS"]
  183 +
  184 + # check if we have all the albums of that layout described in the sequence file
  185 + #with silk_profile(name="Iterate on each album & plan (create)"):
  186 + if len(layout_albums) == len(albums_from_file):
  187 + for album in albums_from_file:
  188 + album = album["Album"]
  189 + if album["name"] not in layout_albums:
  190 + result["errors"].append(f"Album {album['name']} is not the chosen layout")
  191 + else:
  192 + # Create album
  193 + Album.objects.create(name=album["name"], sequence=seq, complete=True)
  194 + # Create plan for that album
  195 + plans = album.get("Plans")
  196 + process_plans(plans, result, is_simplified, config, album, seq)
  197 + else:
  198 + result["errors"].append(f"The number of albums doesn't correspond to the chosen layout")
  199 +
294 200  
  201 +#@silk_profile(name="process_plans")
295 202 def process_plans(plans: dict, result: dict, is_simplified: bool, config: OBSConfig, album: dict, seq: dict):
296 203 if plans == None:
297 204 result["errors"].append(f"Album {album['name']} has no plans. Please add at least one plan")
298 205 # exit function
299 206 return None
300 207 for plan in plans:
301   - new_plan_object = Plan.objects.create(album=Album.objects.get(name=album["name"], sequence=seq), complete=True)
302   - new_plan_object.config_attributes = {}
  208 + #new_plan_object = Plan.objects.create(album=Album.objects.get(name=album["name"], sequence=seq), complete=True)
  209 + #new_plan_object.config_attributes = {}
303 210 plan = plan["Plan"]
  211 + nb_images = 0
304 212 config_attributes = {}
  213 + #with silk_profile(name="Create plan form"):
305 214 plan_form = PlanForm(data_from_config=config.getEditableChannelAttributes(config.unit_name, list(config.get_channels(config.unit_name).keys())[0]), edited_plan=None)
306 215 # Process each plan field
  216 + #with silk_profile(name="iterate on plan fields"):
307 217 for field in plan_form.fields:
308 218 plan_field = plan[field]
309 219 '''
... ... @@ -311,78 +221,187 @@ def process_plans(plans: dict, result: dict, is_simplified: bool, config: OBSCon
311 221 max_value = None
312 222 value_type = None
313 223 '''
314   - min_value = max_value = value_type = None
315   - if field not in plan.keys():
316   - result["errors"].append(f"Missing field : '{field}' for plan {plans.index(plan)}")
317   - continue
318   - # TODO : ajouter max_value, min_value, suppression plan et album si invalides
319   - if not is_simplified:
320   - if plan_field.get("value_type"):
321   - value_type = plan_field["value_type"]
322   - if type(plan_field["value"]) == str and ast.literal_eval(plan_field["value"]) != value_type:
323   - result["errors"].append(f"Field {field} value doesn't correspond to the assigned type (type required : {value_type})")
324   - if plan_field.get("min_value"):
325   - min_value = plan_field["min_value"]
326   - if type(min_value) == str:
327   - min_value = ast.literal_eval(min_value)
328   - '''
329   - if type(plan_field["min_value"]) == str:
330   - min_value = ast.literal_eval(plan_field["min_value"])
331   - else:
332   - min_value = plan_field["min_value"]
333   - '''
334   - if plan_field.get("max_value"):
335   - max_value = plan_field["max_value"]
336   - if type(max_value) == str:
337   - max_value = ast.literal_eval(max_value)
338   - '''
339   - if type(plan_field.get("max_value")) == str:
340   - max_value = ast.literal_eval(plan_field["max_value"])
341   - else:
342   - max_value = plan_field["max_value"]
343   - '''
344 224 if field == "nb_images":
345   - new_plan_object.__dict__[field] = plan_field if is_simplified else plan_field["value"]
346   - '''
347   - if is_simplified:
348   - new_plan_object.__dict__[field] = plan_field
349   - else:
350   - new_plan_object.__dict__[field] = plan_field["value"]
351   - '''
  225 + nb_images = plan_field if is_simplified else plan_field["value"]
352 226 else:
353   - # shortcut possible ?
354   - #new_plan_object_field = new_plan_object.config_attributes[field]
355   - if is_simplified:
356   - new_plan_object.config_attributes[field] = plan_field
357   - else:
358   - if plan_field.get("values"):
359   - index_value = plan_field["value"]
360   - values = plan_field["values"]
361   - if index_value < 0 or index_value > len(plan_field["values"]):
362   - result["errors"].append(f"Value of Plan field '{field}' isn't valid, index out of bounds ({index_value} > {len(values)})")
363   - index_value = 0
364   - value = plan_field["values"][index_value]
365   - try:
366   - # linked values
367   - splitted_values = value.split(";")
368   - config_attributes[field] = {}
369   - for splitted_value in splitted_values:
370   - subkey,subvalue = splitted_value.split(":")
371   - config_attributes[field][subkey] = ast.literal_eval(subvalue)
372   - # vaudrait mieux préciser l'exception ici
373   - except:
374   - # Do nothing, normal string
375   - config_attributes[field] = ast.literal_eval(value)
376   - new_plan_object.config_attributes[field] = config_attributes[field]
377   - else:
378   - if max_value and min_value:
379   - if plan_field["value"] > max_value:
380   - result["errors"].append(f"Plan field {field} doesn't respect max value")
381   - if plan_field["value"] < min_value:
382   - result["errors"].append(f"Plan field {field} doesn't respect min value")
383   - new_plan_object.config_attributes[field] = plan_field["value"]
  227 + process_plan_field(result, config_attributes, plan_field, field, plans, plan, is_simplified)
384 228 # end foreach plan field
385   - new_plan_object.save()
  229 + Plan.objects.create(album=Album.objects.get(name=album["name"], sequence=seq), complete=True, nb_images=nb_images, config_attributes=config_attributes)
  230 +
  231 +def process_plan_field(result, config_attributes, plan_field, field, plans, plan, is_simplified):
  232 +
  233 +
  234 + if field not in plan.keys():
  235 + result["errors"].append(f"Missing field : '{field}' for plan {plans.index(plan)}")
  236 + # exit function
  237 + return None
  238 + if is_simplified:
  239 + #new_plan_object.config_attributes[field] = plan_field
  240 + config_attributes[field] = plan_field
  241 + else:
  242 + value_type, min_value, max_value = prepare_check_plan_field_value(plan_field, field, result)
  243 + check_and_set_plan_field_value(config_attributes, plan_field, field, result, value_type, min_value, max_value)
  244 +
  245 +
  246 +def check_and_set_plan_field_value(config_attributes, plan_field, field, result, value_type, min_value, max_value):
  247 + # if the value is a index of a list, get the value from this index
  248 + if plan_field.get("values"):
  249 + index_value = plan_field["value"]
  250 + values = plan_field["values"]
  251 + if index_value < 0 or index_value > len(plan_field["values"]):
  252 + result["errors"].append(f"Value of Plan field '{field}' isn't valid, index out of bounds ({index_value} > {len(values)})")
  253 + index_value = 0
  254 + value = plan_field["values"][index_value]
  255 + try:
  256 + # linked values
  257 + splitted_values = value.split(";")
  258 + config_attributes[field] = {}
  259 + for splitted_value in splitted_values:
  260 + subkey,subvalue = splitted_value.split(":")
  261 + config_attributes[field][subkey] = ast.literal_eval(subvalue)
  262 + # vaudrait mieux préciser l'exception ici
  263 + except ValueError:
  264 + # Do nothing, normal string
  265 + config_attributes[field] = ast.literal_eval(value)
  266 + #new_plan_object.config_attributes[field] = config_attributes[field]
  267 + else:
  268 + # check min and max values if they exist
  269 + if max_value and min_value:
  270 + if plan_field["value"] > max_value:
  271 + result["errors"].append(f"Plan field {field} doesn't respect max value")
  272 + if plan_field["value"] < min_value:
  273 + result["errors"].append(f"Plan field {field} doesn't respect min value")
  274 + #new_plan_object.config_attributes[field] = plan_field["value"]
  275 + config_attributes[field] = plan_field["value"]
  276 +
  277 +
  278 +def prepare_check_plan_field_value(plan_field, field, result):
  279 + min_value = max_value = value_type = None
  280 + # get value type, min_value and max_value if they're in the plan form
  281 + if plan_field.get("value_type"):
  282 + value_type = plan_field["value_type"]
  283 + # If value type doesn't match with the value from the form, add an error to result
  284 + if type(plan_field["value"]) == str and ast.literal_eval(plan_field["value"]) != value_type:
  285 + result["errors"].append(f"Field {field} value doesn't correspond to the assigned type (type required : {value_type})")
  286 + if plan_field.get("min_value"):
  287 + min_value = plan_field["min_value"]
  288 + if type(min_value) == str:
  289 + min_value = ast.literal_eval(min_value)
  290 + if plan_field.get("max_value"):
  291 + max_value = plan_field["max_value"]
  292 + if type(max_value) == str:
  293 + max_value = ast.literal_eval(max_value)
  294 + return value_type, min_value, max_value
  295 +
  296 +
  297 +def process_sequence(yaml_content, seq, config, is_simplified, result, user_sp):
  298 +
  299 + # From user sp, get all SP that can observe / submit sequence for the current period
  300 + sp_list = ScientificProgram.objects.observable_programs().filter(id__in=user_sp)
  301 + # Create a Sequence form
  302 + sequence_form = SequenceForm(instance=seq, data_from_config=config.getEditableMountAttributes(config.unit_name), layouts = config.get_layouts(config.unit_name), sp_list=sp_list)
  303 +
  304 + if is_simplified:
  305 + seq.scientific_program = sp_list[yaml_content["sequence"]["scientific_program"]]
  306 + else:
  307 + # get scientific program field's attributes
  308 + yaml_seq_sp = yaml_content["sequence"]["scientific_program"]
  309 + sp_index_value = yaml_seq_sp["value"]
  310 + values = yaml_seq_sp["values"]
  311 + # Check if index of sp is valid (in range of possible index from values)
  312 + if sp_index_value < 0 or sp_index_value > len(values):
  313 + result["errors"].append(f"SP value isn't valid, index out of bounds ({sp_index_value} > {len(values)})")
  314 + sp_index_value = 0
  315 + chosen_sp = ScientificProgram.objects.get(name=values[sp_index_value])
  316 + # If the sp is associated to that user, associate the sp to the sequence
  317 + if chosen_sp in sp_list:
  318 + #seq.scientific_program = ScientificProgram.objects.get(name=yaml_content["sequence"]["scientific_program"]["values"][sp_index_value])
  319 + seq.scientific_program = chosen_sp
  320 + else:
  321 + result["errors"].append(f"SP {chosen_sp.name} is not assigned to that user ")
  322 +
  323 + seq.config_attributes = {}
  324 +
  325 + # Fill all Sequence form fields
  326 + #with silk_profile(name="iterate sequence fields form"):
  327 + for field, field_attributes in sequence_form.fields.items():
  328 + #if sequence_form.fields[field].required == False or field == "scientific_program":
  329 + if not field_attributes.required or field=="scientific_program":
  330 + continue
  331 + # pour lisibilité, simplicité et éviter redondance
  332 + yaml_field = yaml_content["sequence"][field]
  333 + value = yaml_field if is_simplified else yaml_field["value"]
  334 + ''' (orig)
  335 + if is_simplified:
  336 + value = yaml_content["sequence"][field]
  337 + else:
  338 + value = yaml_content["sequence"][field]["value"]
  339 + '''
  340 + # If the current field of the sequence isn't found in the file, add an error message to inform the user the missing field
  341 + if field not in yaml_content["sequence"]:
  342 + result["errors"].append(f"{field} not in yaml file")
  343 + else:
  344 + if is_simplified:
  345 + # If the field is a choicefield, get choices and associate the index to the real value
  346 + if sequence_form.fields[field].__dict__.get("_choices"):
  347 + # y a pas conflit ici avec la variable "value" définie au-dessus ? -> Non car on transforme l'ancien value qui est un index en une vraie valeur
  348 + values = [value[0] for value in sequence_form.fields[field].__dict__.get("_choices")]
  349 + value = values[value]
  350 + # Transform the string value to a datetime value
  351 + if field == "start_date":
  352 + if type(value) != datetime.datetime:
  353 + #value = datetime.datetime.strptime(yaml_content["sequence"][field]["value"],'%d/%m/%Y %H:%M:%S')
  354 + # ISO format
  355 + value = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%f')
  356 + seq.__dict__[field] = value
  357 + else:
  358 + if yaml_field.get("values"):
  359 + # Transform the original value which is an index to a "real" value from the "values" attributes
  360 + index_value = yaml_field["value"]
  361 + values = yaml_field["values"]
  362 + if index_value < 0 or index_value > len(yaml_field["values"]):
  363 + result["errors"].append(f"Value of {field} isn't valid, index out of bounds ({index_value} > {len(values)})")
  364 + index_value = 0
  365 + value = yaml_field["values"][index_value]
  366 + else:
  367 + # Transform the string value to a datetime value
  368 + if field == "start_date":
  369 + if type(value) != datetime.datetime:
  370 + #value = datetime.datetime.strptime(yaml_content["sequence"][field]["value"],'%d/%m/%Y %H:%M:%S')
  371 + # ISO format
  372 + value = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%f')
  373 + seq.__dict__[field] = value
  374 + ''' (orig)
  375 + else:
  376 + if not is_simplified:
  377 + if yaml_content["sequence"][field].get("values"):
  378 + index_value = yaml_content["sequence"][field]["value"]
  379 + values = yaml_content["sequence"][field]["values"]
  380 + if index_value < 0 or index_value > len(yaml_content["sequence"][field]["values"]):
  381 + result["errors"].append(f"Value of {field} isn't valid, index out of bounds ({index_value} > {len(values)})")
  382 + index_value = 0
  383 + value = yaml_content["sequence"][field]["values"][index_value]
  384 + else:
  385 + if field == "start_date":
  386 + if type(value) != datetime.datetime:
  387 + #value = datetime.datetime.strptime(yaml_content["sequence"][field]["value"],'%d/%m/%Y %H:%M:%S')
  388 + # ISO format
  389 + value = datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%f')
  390 + seq.__dict__[field] = value
  391 + else:
  392 + if sequence_form.fields[field].__dict__.get("_choices"):
  393 + values = [value[0] for value in sequence_form.fields[field].__dict__.get("_choices")]
  394 + value = values[value]
  395 + '''
  396 +
  397 + # suffisant ? => if field in seq.__dict__
  398 + # If field is an attribute of the sequence, associate the field to the value
  399 + if field in seq.__dict__:
  400 + seq.__dict__[field] = value
  401 + else:
  402 + # else associate field & value in config_attributes sequence's field (JsonField) = variable fields of an sequence
  403 + seq.config_attributes[field] = value
  404 +
386 405  
387 406 def create_sequence_pickle(sequence):
388 407 seq_dict = model_to_dict(sequence)
... ... @@ -404,4 +423,6 @@ def create_sequence_pickle(sequence):
404 423 if not os.path.exists(data_path +f"sequences_pickle/P{period.id}/{sequence.night_id}"):
405 424 os.mkdir(data_path +f"sequences_pickle/P{period.id}/{sequence.night_id}")
406 425 seq_pickle_file_name = data_path +f"./sequences_pickle/P{period.id}/{sequence.night_id}/{sequence.id}.p"
  426 + # get guitastro ephemeris
  427 + #fullseq_dict["ephem"] = fn.ephem(sequence.target)
407 428 pickle.dump(fullseq_dict,open(seq_pickle_file_name,"wb"))
408 429 \ No newline at end of file
... ...
src/core/pyros_django/routine_manager/templates/routine_manager/view_sequence.html
  Diff comments   View file @76a6bf8
... ... @@ -44,14 +44,14 @@
44 44 <td>&nbsp</td>
45 45 <td><button type="submit" class="btn btn-success" name="action" value="check_validity">Check validity</button></td>
46 46 <td>&nbsp</td>
47   - {% if seq.complete and seq.scientific_program in request.user.get_scientific_program and seq.status == "DRAFT" %}
  47 + {% if seq.complete and seq.scientific_program in request.user.get_scientific_programs and seq.status == "DRAFT" %}
48 48 <td><button type="submit" class="btn btn-success" name="action" value="save_and_submit">Save and Submit</button></td>
49 49 <td>&nbsp</td>
50 50 {% else %}
51 51 <td><button type="submit" class="btn btn-success" name="action" value="save_and_submit" disabled>Save and Submit</button></td>
52 52 <td>&nbsp</td>
53 53 {% endif %}
54   - {% if seq.status == "TBP" and seq.scientific_program in request.user.get_scientific_program %}
  54 + {% if seq.status == "TBP" and seq.scientific_program in request.user.get_scientific_programs %}
55 55  
56 56 <td><a href="{% url "unsubmit_sequence" seq.id %}" class="btn btn-warning" onclick="return confirm('The sequence will be unsubmitted')"
57 57 title="The sequence will be unsubmitted">Unsubmit sequence</a></td>
... ...
src/core/pyros_django/routine_manager/tests.py
  Diff comments   View file @76a6bf8
... ... @@ -268,9 +268,9 @@ class SequencesTests(TestCase):
268 268 pyros_user=self.usr1).order_by("-created").first()
269 269 seq_id = created_seq.id
270 270 config = OBSConfig(os.environ["PATH_TO_OBSCONF_FILE"])
271   - sp_of_user = self.usr1.get_scientific_program()
  271 + sp_of_user = self.usr1.get_scientific_programs()
272 272 sp_list = ScientificProgram.objects.observable_programs().filter(id__in=sp_of_user)
273   - sequence_form = SequenceForm(instance=created_seq, data_from_config=config.getEditableAttributesOfMount(
  273 + sequence_form = SequenceForm(instance=created_seq, data_from_config=config.getEditableMountAttributes(
274 274 config.unit_name), layouts=config.get_layouts(config.unit_name), sp_list=sp_list)
275 275 layout = list(config.get_layouts(config.unit_name)
276 276 ["layouts"].values())[0]["name"]
... ... @@ -317,7 +317,7 @@ class SequencesTests(TestCase):
317 317 # but this plan doesn't have config_attributes, we need to add them
318 318 plan = Plan.objects.get(album=album)
319 319 post_data = {}
320   - plan_form = PlanForm(data_from_config=config.getEditableAttributesOfChannel(
  320 + plan_form = PlanForm(data_from_config=config.getEditableChannelAttributes(
321 321 config.unit_name, list(config.get_channels(config.unit_name).keys())[0]), edited_plan=None)
322 322 for field_name, field in plan_form.fields.items():
323 323 if field.__dict__.get("_choices"):
... ... @@ -359,9 +359,9 @@ class SequencesTests(TestCase):
359 359 pyros_user=self.usr1).order_by("-created").first()
360 360 seq_id = created_seq.id
361 361 config = OBSConfig(os.environ["PATH_TO_OBSCONF_FILE"])
362   - sp_of_user = self.usr1.get_scientific_program()
  362 + sp_of_user = self.usr1.get_scientific_programs()
363 363 sp_list = ScientificProgram.objects.observable_programs().filter(id__in=sp_of_user)
364   - sequence_form = SequenceForm(instance=created_seq, data_from_config=config.getEditableAttributesOfMount(
  364 + sequence_form = SequenceForm(instance=created_seq, data_from_config=config.getEditableMountAttributes(
365 365 config.unit_name), layouts=config.get_layouts(config.unit_name), sp_list=sp_list)
366 366 layout = list(config.get_layouts(config.unit_name)
367 367 ["layouts"].values())[0]["name"]
... ...
src/core/pyros_django/user_manager/models.py
  Diff comments   View file @76a6bf8
... ... @@ -219,12 +219,13 @@ class PyrosUser(AbstractUser):
219 219 else:
220 220 return str
221 221  
222   - def get_scientific_program(self) -> QuerySet:
  222 + def get_scientific_programs(self) -> QuerySet:
223 223 sp_where_user_is_sp_pi = ScientificProgram.objects.filter(
224 224 sp_pi=self.id)
225   - other_sp_of_user = ScientificProgram.objects.filter(id__in=SP_Period.objects.filter(id__in=SP_Period_User.objects.filter(
  225 + # Get all SP of user where he's not an SP PI
  226 + user_other_sp = ScientificProgram.objects.filter(id__in=SP_Period.objects.filter(id__in=SP_Period_User.objects.filter(
226 227 user=PyrosUser.objects.get(username=self.username)).values("SP_Period")).values_list("scientific_program", flat=True))
227   - sp_of_user = sp_where_user_is_sp_pi | other_sp_of_user
  228 + sp_of_user = sp_where_user_is_sp_pi | user_other_sp
228 229 return sp_of_user
229 230  
230 231 def get_scientific_program_where_user_is_sp_pi(self) -> QuerySet:
... ...
src/core/pyros_django/user_manager/views.py
  Diff comments   View file @76a6bf8
... ... @@ -229,7 +229,7 @@ def users(request):
229 229 inactive_pyros_users = PyrosUser.objects.filter(
230 230 is_active=False).order_by("-id")
231 231 else:
232   - sp_of_current_user = current_user.get_scientific_program()
  232 + sp_of_current_user = current_user.get_scientific_programs()
233 233 sp_periods_of_current_user = SP_Period.objects.filter(
234 234 scientific_program__in=sp_of_current_user)
235 235 common_scientific_programs = sp_of_current_user
... ... @@ -325,7 +325,7 @@ def user_detail_view(request, pk):
325 325 "role") in ("Admin", "Unit-PI", "Unit-board")
326 326 CAN_VIEW_MOTIVE_OF_REGISTRATION = request.session.get("role") in (
327 327 "Admin", "Unit-PI", "Unit-board") and len(user.motive_of_registration) > 0
328   - scientific_programs = user.get_scientific_program()
  328 + scientific_programs = user.get_scientific_programs()
329 329 except PyrosUser.DoesNotExist:
330 330 raise Http404("User does not exist")
331 331 return render(request, 'user_manager/user_detail.html', context={
... ...
upload_sequence.md 0 → 100644
  Diff comments   View file @76a6bf8
... ... @@ -0,0 +1,20 @@
  1 +```mermaid
  2 +flowchart TD
  3 + A[Receive yaml file]-->B[Load yaml file]
  4 + B-->id1
  5 + subgraph id1 [Transcript file into Sequence]
  6 + subgraph id2 [Process sequence]
  7 + C[Get scientific program of Sequence]-->D[Process form fields]
  8 + end
  9 + id2-->id3
  10 + subgraph id3 [Process albums]
  11 + F[Get album name]-->G[Create album]
  12 + end
  13 + id3-->id4
  14 + subgraph id4 [Process plan]
  15 + H[Process plan fields]-->I[Create plan]
  16 + end
  17 + end
  18 + id1-->J[Create Sequence]
  19 + J-->K[Return sequence id]
  20 +```
0 21 \ No newline at end of file
... ...