PYROS-IRAP / pyros

Commit 675fb3d51bcc4c38d7903e8de53a279258c5199f

Authored by Jeremy
1 parent bca9a283
Exists in master and in 1 other branch dev

Update scheduler / simulator

Showing 19 changed files with 173 additions and 65 deletions   Show diff stats
Inline Side-by-side
pyros.py
  Diff comments   View file @675fb3d
... ... @@ -491,6 +491,46 @@ class Pyros(AManager):
491 491 self.makemigrations()
492 492 self.migrate()
493 493 self.loaddata()
  494 + return 0
  495 +
  496 + def simulator_development(self):
  497 + self.changeDirectory("src")
  498 + self.replacePatternInFile("CELERY_TEST = False", "CELERY_TEST = True", "pyros/settings.py")
  499 + self.execProcess("rm -f testdb.sqlite3")
  500 + self.changeDirectory("..")
  501 + self.migrate()
  502 + self.loaddata()
  503 + self.server()
  504 + self.sleep(2)
  505 + self.printFullTerm(Colors.WARNING, "SUMMARY")
  506 + self.printColor(Colors.GREEN, "The simulator has been successfully initialised")
  507 + self.printColor(Colors.GREEN, "The simulator run on a temp database : src/testdb.sqlite3")
  508 + self.printColor(Colors.GREEN, "The simulation will be ended by the task 'simulator herself'")
  509 + self.printColor(Colors.GREEN, "If you want to shutdown the simulation, please run :")
  510 + self.printColor(Colors.GREEN, "CTRL-C or ./pyrosrun.sh kill_simulation")
  511 + self.printColor(Colors.GREEN, "If the simulation isn't correctly killed, please switch the variable")
  512 + self.printColor(Colors.GREEN, "CELERY_TEST in src/pyros/settings.py to false")
  513 + self.printFullTerm(Colors.WARNING, "SUMMARY")
  514 + self.changeDirectory("simulators/config")
  515 + self.printColor(Colors.BOLD, "Existing simulations : ", eol='')
  516 + sys.stdout.flush()
  517 + self.execProcessSilent("ls conf*.json")
  518 + self.changeDirectory("..")
  519 + conf = self.askQuestion("Which simulation do you want to use", default="conf.json")
  520 + self.changeDirectory("..")
  521 + self.singleWorker("scheduling")
  522 + self.singleWorker("majordome")
  523 + self.sleep(3)
  524 + procs = []
  525 + self.changeDirectory("simulators")
  526 + self.changeDirectory("user")
  527 + procs.append(self.execProcessFromVenvAsync(self.venv_bin + " userSimulator.py " + conf))
  528 + self.changeDirectory("..")
  529 + for p in procs:
  530 + p.wait()
  531 + self.changeDirectory("..")
  532 + self.kill_simulation()
  533 + return 0
494 534  
495 535 def simulator(self):
496 536 self.changeDirectory("src")
... ... @@ -598,6 +638,7 @@ class Pyros(AManager):
598 638 "update": self.update,
599 639 "server": self.server,
600 640 "clean": self.clean,
  641 + "simulator_scheduler": self.simulator_development,
601 642 "clean_logs": self.clean_logs,
602 643 "test": self.test,
603 644 "migrate": self.migrate,
... ... @@ -636,6 +677,7 @@ class Pyros(AManager):
636 677 "start": "Stop the celery workers then the web server",
637 678 "stop": "stops the celery workers",
638 679 "simulator": "Launch a simulation",
  680 + "simulator_scheduler": "Simulation for the scheduler only",
639 681 "kill_simulation": "kill the simulators / celery workers / web server",
640 682 "sims_launch": "Launch only the simulators",
641 683 }
... ...
simulators/config/conf.json
  Diff comments   View file @675fb3d
... ... @@ -4,24 +4,40 @@
4 4 "userSimulator" : "routine_request_01.xml"
5 5 },
6 6 {
7   - "time" : 23,
  7 + "time" : 2,
8 8 "userSimulator" : "routine_request_02.xml"
9 9 },
10 10 {
11   - "time" : 24,
  11 + "time" : 4,
12 12 "userSimulator" : "routine_request_03.xml"
13 13 },
14 14 {
15   - "time" : 24,
  15 + "time" : 5,
16 16 "userSimulator" : "routine_request_04.xml"
17 17 },
18 18 {
19   - "time" : 1,
20   - "alertSimulator" : "voevent_alert_04.xml"
  19 + "time" : 10,
  20 + "userSimulator" : "routine_request_05.xml"
21 21 },
22 22 {
23   - "time" : 30,
24   - "userSimulator" : "routine_request_05.xml"
  23 + "time" : 11,
  24 + "userSimulator" : "routine_request_06.xml"
  25 + },
  26 + {
  27 + "time" : 15,
  28 + "userSimulator" : "routine_request_07.xml"
  29 + },
  30 + {
  31 + "time" : 16,
  32 + "userSimulator" : "routine_request_08.xml"
  33 + },
  34 + {
  35 + "time" : 18,
  36 + "userSimulator" : "routine_request_09.xml"
  37 + },
  38 + {
  39 + "time" : 33,
  40 + "userSimulator" : "routine_request_10.xml"
25 41 },
26 42 {
27 43 "time" : 35,
... ...
simulators/resources/routine_request_01.xml
  Diff comments   View file @675fb3d
1 1 <?xml version="1.0" ?>
2 2 <request submitted="1" relative="1" name="RequestSimulator" scientific_program="GRB" target_type="test">
3   - <sequence duration="19.99987200" jd1="6000" jd2="18000" name="Seq" target_coords="10">
  3 + <sequence duration="200" jd1="6000" jd2="18000" name="Sequence1 (200secs)" target_coords="10">
4 4 <album detector="Visible camera" name="alb">
5   - <plan duration="19.999999999999957" filter="First infrared filter" name="New plan" nb_images="5"/>
  5 + <plan duration="200" filter="First infrared filter" name="New plan" nb_images="5"/>
6 6 </album>
7 7 </sequence>
8 8 </request>
... ...
simulators/resources/routine_request_02.xml
  Diff comments   View file @675fb3d
1 1 <?xml version="1.0" ?>
2 2 <request submitted="1" relative="1" name="RequestSimulator" scientific_program="GRB" target_type="test">
3   - <sequence duration="19.99987200" jd1="7000" jd2="19000" name="Seq" target_coords="10">
  3 + <sequence duration="100" jd1="6000" jd2="18000" name="Sequence2 (100 secs)" target_coords="10">
4 4 <album detector="Visible camera" name="alb">
5   - <plan duration="19.999999999999957" filter="First infrared filter" name="New plan" nb_images="5"/>
  5 + <plan duration="100" filter="First infrared filter" name="New plan" nb_images="5"/>
6 6 </album>
7 7 </sequence>
8 8 </request>
... ...
simulators/resources/routine_request_03.xml
  Diff comments   View file @675fb3d
1 1 <?xml version="1.0" ?>
2 2 <request submitted="1" relative="1" name="RequestSimulator" scientific_program="GRB" target_type="test">
3   - <sequence duration="19.99987200" jd1="10000" jd2="15000" name="Seq" target_coords="10">
  3 + <sequence duration="40" jd1="10000" jd2="15000" name="Sequence3 (40secs)" target_coords="10">
4 4 <album detector="Visible camera" name="alb">
5   - <plan duration="19.999999999999957" filter="First infrared filter" name="New plan" nb_images="5"/>
  5 + <plan duration="40" filter="First infrared filter" name="New plan" nb_images="5"/>
6 6 </album>
7 7 </sequence>
8 8 </request>
... ...
simulators/resources/routine_request_04.xml
  Diff comments   View file @675fb3d
1 1 <?xml version="1.0" ?>
2 2 <request submitted="1" relative="1" name="RequestSimulator" scientific_program="GRB" target_type="test">
3   - <sequence duration="19.99987200" jd1="12000" jd2="25000" name="Seq" target_coords="10">
  3 + <sequence duration="500" jd1="12000" jd2="25000" name="Sequence 4 (500secs)" target_coords="10">
4 4 <album detector="Visible camera" name="alb">
5   - <plan duration="19.999999999999957" filter="First infrared filter" name="New plan" nb_images="5"/>
  5 + <plan duration="500" filter="First infrared filter" name="New plan" nb_images="5"/>
6 6 </album>
7 7 </sequence>
8 8 </request>
... ...
simulators/resources/routine_request_05.xml
  Diff comments   View file @675fb3d
1 1 <?xml version="1.0" ?>
2 2 <request submitted="1" relative="1" name="RequestSimulator" scientific_program="GRB" target_type="test">
3   - <sequence duration="19.99987200" jd1="50" jd2="150" name="Seq" target_coords="10">
  3 + <sequence duration="10" jd1="6000" jd2="22000" name="Sequence5 (10secs)" target_coords="10">
4 4 <album detector="Visible camera" name="alb">
5   - <plan duration="19.999999999999957" filter="First infrared filter" name="New plan" nb_images="5"/>
  5 + <plan duration="10" filter="First infrared filter" name="plan vis" nb_images="1"/>
6 6 </album>
7 7 </sequence>
8 8 </request>
... ...
simulators/resources/routine_request_06.xml 0 → 100644
  Diff comments   View file @675fb3d
... ... @@ -0,0 +1,8 @@
  1 +<?xml version="1.0" ?>
  2 +<request submitted="1" relative="1" name="RequestSimulator" scientific_program="GRB" target_type="test">
  3 + <sequence duration="200" jd1="6000" jd2="18000" name="Sequence6 (200 secs)" target_coords="10">
  4 + <album detector="Visible camera" name="alb">
  5 + <plan duration="200" filter="First infrared filter" name="New plan" nb_images="5"/>
  6 + </album>
  7 + </sequence>
  8 +</request>
... ...
simulators/resources/routine_request_07.xml 0 → 100644
  Diff comments   View file @675fb3d
... ... @@ -0,0 +1,8 @@
  1 +<?xml version="1.0" ?>
  2 +<request submitted="1" relative="1" name="RequestSimulator" scientific_program="GRB" target_type="test">
  3 + <sequence duration="200" jd1="6000" jd2="18000" name="Sequence7 (200secs)" target_coords="10">
  4 + <album detector="Visible camera" name="alb">
  5 + <plan duration="200" filter="First infrared filter" name="New plan" nb_images="5"/>
  6 + </album>
  7 + </sequence>
  8 +</request>
... ...
simulators/resources/routine_request_08.xml 0 → 100644
  Diff comments   View file @675fb3d
... ... @@ -0,0 +1,8 @@
  1 +<?xml version="1.0" ?>
  2 +<request submitted="1" relative="1" name="RequestSimulator" scientific_program="GRB" target_type="test">
  3 + <sequence duration="200" jd1="6000" jd2="18000" name="Sequence8 (200secs)" target_coords="10">
  4 + <album detector="Visible camera" name="alb">
  5 + <plan duration="200" filter="First infrared filter" name="New plan" nb_images="5"/>
  6 + </album>
  7 + </sequence>
  8 +</request>
... ...
simulators/resources/routine_request_09.xml 0 → 100644
  Diff comments   View file @675fb3d
... ... @@ -0,0 +1,8 @@
  1 +<?xml version="1.0" ?>
  2 +<request submitted="1" relative="1" name="RequestSimulator" scientific_program="GRB" target_type="test">
  3 + <sequence duration="200" jd1="6000" jd2="18000" name="Sequence9 (200secs)" target_coords="10">
  4 + <album detector="Visible camera" name="alb">
  5 + <plan duration="200" filter="First infrared filter" name="New plan" nb_images="5"/>
  6 + </album>
  7 + </sequence>
  8 +</request>
... ...
simulators/resources/routine_request_10.xml 0 → 100644
  Diff comments   View file @675fb3d
... ... @@ -0,0 +1,8 @@
  1 +<?xml version="1.0" ?>
  2 +<request submitted="1" relative="1" name="RequestSimulator" scientific_program="GRB" target_type="test">
  3 + <sequence duration="200" jd1="6000" jd2="18000" name="Sequence10 (200secs)" target_coords="10">
  4 + <album detector="Visible camera" name="alb">
  5 + <plan duration="200" filter="First infrared filter" name="New plan" nb_images="5"/>
  6 + </album>
  7 + </sequence>
  8 +</request>
... ...
simulators/user/userSimulator.py
  Diff comments   View file @675fb3d
... ... @@ -68,7 +68,7 @@ class UserSimulator():
68 68 if self.authenticate():
69 69 return 1
70 70 self.userPrint("The simulator will end in %d seconds"%(int(self.ended)))
71   - while (i < self.ended):
  71 + while (i <= self.ended):
72 72 for dic in self.sims:
73 73 if (int(dic["time"]) == i):
74 74 self.sendRequest(dic["userSimulator"])
... ...
src/pyros/settings.py
  Diff comments   View file @675fb3d
... ... @@ -119,6 +119,8 @@ LOGIN_URL = &quot;/&quot;
119 119  
120 120 # EP modif
121 121  
  122 +SIMULATOR = False
  123 +
122 124 CELERY_TEST = False
123 125  
124 126 if not CELERY_TEST:
... ...
src/pyros/settings.py.bak
  Diff comments   View file @675fb3d
... ... @@ -119,7 +119,7 @@ LOGIN_URL = &quot;/&quot;
119 119  
120 120 # EP modif
121 121  
122   -CELERY_TEST = False
  122 +CELERY_TEST = True
123 123  
124 124 if not CELERY_TEST:
125 125 if not MYSQL:
... ...
src/scheduler/Interval.py
  Diff comments   View file @675fb3d
... ... @@ -14,7 +14,7 @@ class Interval:
14 14 def __init__(self, start, end):
15 15 self._start = Decimal(start)
16 16 self._end = Decimal(end)
17   - self.duration = Decimal(end - start)
  17 + self.duration = self._end - self._start
18 18  
19 19 def __str__(self):
20 20 print("[" + str(self.start) + " - " + str(self.end) + "]")
... ... @@ -45,7 +45,7 @@ class Interval:
45 45 class IntervalManagement(Logger):
46 46 def __init__(self, name: str = "IntervalManagement", file: str = "IntervalManagement"):
47 47 super().__init__(name, file)
48   - self.max_overhead_seconds = 25
  48 + self.max_overhead_seconds = 10
49 49 self.max_overhead = Decimal(self.max_overhead_seconds / JD_VALUE)
50 50 self.intervals = []
51 51  
... ...
src/scheduler/Scheduler.py
  Diff comments   View file @675fb3d
... ... @@ -24,24 +24,22 @@ class Scheduler(IntervalManagement):
24 24 def setNightLimits(self, plan_start: float, plan_end: float) -> int:
25 25 self.schedule.plan_start = Decimal(plan_start)
26 26 self.schedule.plan_end = Decimal(plan_end)
27   -
28   - self.log("Schedule plan start -> " + str(plan_start))
29   - self.log("Schedule plan end -> " + str(plan_end))
30 27 return 0
31 28  
32 29 def isFirstSchedule(self) -> bool:
33 30 return False
34 31  
35 32 def copyFromPrevious(self) -> int:
36   - if not Schedule.objects.exists():
  33 + if len(Schedule.objects.all()) == 1:
37 34 self.schedule.plan_night_start = self.schedule.plan_start
  35 + self.log("No schedule found")
38 36 return 1
39 37 try:
40 38 previous_sched = Schedule.objects.order_by('-created')[1]
41 39 previous_exc_seq = previous_sched.sequences.filter(status=Sequence.EXECUTED)
42 40 except:
43 41 self.schedule.plan_night_start = self.schedule.plan_start
44   - self.debug("Scheduler could not get informations from previous schedule")
  42 + self.debug("Scheduler could not get information from previous schedule")
45 43 return 1
46 44 for seq in previous_exc_seq:
47 45 shs = seq.shs
... ... @@ -50,7 +48,8 @@ class Scheduler(IntervalManagement):
50 48 shs.save()
51 49 self.schedule.plan_night_start = previous_sched.plan_night_start
52 50 self.schedule.plan_end = previous_sched.plan_end
53   - self.schedule.plan_start = Decimal(secondsToJulianDate(getCurrentTime())) + self.max_overhead
  51 + start = Decimal(secondsToJulianDate(getCurrentTime())) + self.max_overhead
  52 + self.schedule.plan_start = (start if start > previous_sched.plan_start else previous_sched.plan_start)
54 53 return 0
55 54  
56 55 def simulateSchedule(self, sequences) -> tuple:
... ... @@ -77,6 +76,34 @@ class Scheduler(IntervalManagement):
77 76 self.placeSequences()
78 77 return 0
79 78  
  79 + def makeSchedule(self) -> Schedule:
  80 + global SIMULATION
  81 + SIMULATION = False
  82 +
  83 + if self.isFirstSchedule():
  84 + self.schedule.plan_night_start = self.schedule.plan_start
  85 + else:
  86 + if self.copyFromPrevious():
  87 + self.schedule.plan_night_start = self.schedule.plan_start
  88 +
  89 + self.sequences = list(Sequence.objects.filter())
  90 + self.sequences = [(sequence, ScheduleHasSequences(sequence=sequence, schedule=self.schedule))
  91 + for sequence in self.sequences]
  92 + self.log(str(len(self.sequences)) + " sequences found")
  93 + self.computeSchedule()
  94 + self.saveSchedule()
  95 + self.log("Saving schedule with " + str(len(self.schedule.sequences.all())) + " sequences")
  96 + return self.schedule
  97 +
  98 + def saveSchedule(self) -> int:
  99 + self.schedule.save()
  100 + for _, shs in self.sequences:
  101 + shs.schedule = self.schedule
  102 + shs.save()
  103 + self.logSchedule()
  104 + return 0
  105 +
  106 +
80 107 '''
81 108 JB: Using : list(self.sequences) makes a copy.
82 109 '''
... ... @@ -233,7 +260,7 @@ class Scheduler(IntervalManagement):
233 260 '''
234 261 def sortSequences(self) -> int:
235 262 self.sequences.sort(key=lambda x: x[0].jd2)
236   - self.sequences.sort(key=lambda x: x[0].priority)
  263 + self.sequences.sort(key=lambda x: x[0].priority if x[0].priority else 0)
237 264 return 0
238 265  
239 266 def determinePriorities(self) -> int:
... ... @@ -245,49 +272,28 @@ class Scheduler(IntervalManagement):
245 272 return 0
246 273 return user.decreaseQuota(Decimal(quota))
247 274  
248   - def makeSchedule(self) -> Schedule:
249   - global SIMULATION
250   - SIMULATION = False
251   -
252   - if self.isFirstSchedule():
253   - self.schedule.plan_night_start = self.schedule.plan_start
254   - else:
255   - if self.copyFromPrevious():
256   - self.schedule.plan_night_start = self.schedule.plan_start
257   -
258   - self.sequences = list(Sequence.objects.filter(status=Sequence.OBSERVABLE))
259   - self.sequences = [(sequence, ScheduleHasSequences(sequence=sequence, schedule=self.schedule))
260   - for sequence in self.sequences]
261   - self.log("There is : " + str(len(self.sequences)) + " sequences")
262   - self.computeSchedule()
263   - self.saveSchedule()
264   - self.log("Saving schedule with " + str(len(self.schedule.sequences.all())) + " sequences")
265   - return self.schedule
266   -
267   - def saveSchedule(self) -> int:
268   - if self.isEmptySchedule():
269   - self.log("Schedule is empty")
270   - return 1
271   - self.schedule.save()
272   - for sequence, shs in self.sequences:
273   - shs.schedule = self.schedule
274   - shs.save()
275   -
276   - self.logSchedule()
277   - return 0
278   -
279 275 def isEmptySchedule(self) -> bool:
280 276 if len(self.sequences) == 0:
281 277 return True
282 278 return False
283 279  
  280 + def logSequence(self, sequence):
  281 + self.log("Logging sequence : ")
  282 + s = sequence.shs.latest("schedule__created")
  283 + if s.schedule == self.schedule:
  284 + self.log("--> name: %r, start: %f, end: %f, duration: %f, deltaTL: %f, deltaTR: %f"
  285 + % (sequence.name, s.tsp, s.tep, sequence.duration, s.deltaTL, s.deltaTR))
  286 + self.log("------ end ------")
  287 + return 0
  288 +
284 289 def logSchedule(self) -> int:
285   - sequences = Sequence.objects.filter(shs__status=Sequence.PENDING).order_by('shs__tsp')
  290 + sequences = Sequence.objects.filter(shs__status=Sequence.PENDING).order_by('shs__tsp').distinct()
286 291 self.log("There are %d sequence(s) planned" % len(sequences))
287 292 for sequence in sequences:
288   - self.log("--> name: %r, start: %d, end: %d, duration: %d, deltaTL: %d, deltaTR: %d"
289   - % (sequence.name, sequence.shs.get().tsp, sequence.shs.get().tep, sequence.duration, sequence.shs.get().deltaTL, sequence.shs.get().deltaTR))
  293 + s = sequence.shs.latest("schedule__created")
  294 + self.log("--> Pk: %d name: %r, shs PK: %d, start: %f, end: %f, duration: %f, deltaTL: %f, deltaTR: %f"
  295 + % (sequence.pk, sequence.name, s.pk, s.tsp, s.tep, sequence.duration, s.deltaTL, s.deltaTR))
290 296 self.log("There are %d free intervals" % len(self.intervals))
291 297 for interval in self.intervals:
292   - self.log("--> start: %d, end: %d" % (interval.start, interval.end))
  298 + self.log("--> start: %f, end: %f" % (interval.start, interval.end))
293 299 return 0
... ...
src/scheduler/UserManager.py
  Diff comments   View file @675fb3d
... ... @@ -26,5 +26,4 @@ class UserManager(Logger):
26 26 def decreaseQuota(self, value: Decimal) -> int:
27 27 self.user.quota -= float(value)
28 28 self.user.save()
29   - self.log("Quota decreased %s for user %s" % (str(value), self.user.username))
30 29 return 0
... ...
src/scheduler/tasks.py
  Diff comments   View file @675fb3d
... ... @@ -3,14 +3,17 @@ from celery.task import Task
3 3 from scheduler.Scheduler import Scheduler
4 4 from common.models import *
5 5 from utils.JDManipulator import *
6   -
  6 +from utils.Logger import setupLogger
  7 +log = setupLogger("TaskSched", "TaskSched")
7 8  
8 9 class scheduling(Task):
9 10  
10 11 def run(self, first_schedule=False, alert=False):
11 12  
12 13 Log.objects.create(agent='Scheduler', message='Start schedule : ' + str(datetime.datetime.now()))
  14 + log.info("Scheduling start")
13 15 self.scheduler = Scheduler()
14 16 self.scheduler.setNightLimits(secondsToJulianDate(getNightStart()), secondsToJulianDate(getNightEnd()))
15 17 self.scheduler.makeSchedule()
  18 + log.info("Scheduling done")
16 19 Log.objects.create(agent='Scheduler', message='Scheduling finished : ' + str(datetime.datetime.now()))
17 20 \ No newline at end of file
... ...