PYROS-IRAP / pyros

Commit d1e6236c783e7d0097f3dc52764d169da0c56c4d

Authored by Etienne Pallier
1 parent c61d3f82
Exists in dev

Refactorized pyros.py : sims_launch, simulator, simulator_development 

Updated doc
Showing 5 changed files with 101 additions and 69 deletions   Show diff stats
Inline Side-by-side
pyros.py
  Diff comments   View file @d1e6236
... ... @@ -9,7 +9,7 @@ import argparse
9 9 import time
10 10 import signal
11 11  
12   -DEBUG = False
  12 +DEBUG = True
13 13  
14 14 '''
15 15 Pyros Manager, able to launch processes and handle all project commands
... ... @@ -467,12 +467,14 @@ class Pyros(AManager):
467 467 self.unittest()
468 468 return 0
469 469  
  470 + #TODO: mettre la fixture en date naive (sans time zone)
470 471 def loaddata(self):
471 472 self.changeDirectory("src")
472 473 self.execProcessFromVenv(self.venv_bin + " manage.py loaddata misc" + os.sep + "fixtures" + os.sep + self.init_fixture)
473 474 self.changeDirectory("..")
474 475 return 0
475 476  
  477 +
476 478 def celery_on(self, TOTAL=True):
477 479 self.changeDirectory("src")
478 480  
... ... @@ -624,10 +626,12 @@ class Pyros(AManager):
624 626 self.replacePatternInFile("CELERY_TEST = False", "CELERY_TEST = True", "pyros/settings.py")
625 627 self.changeDirectory("..")
626 628  
  629 +
627 630 # Simulation for the scheduler ONLY (and only some celery workers, not all of them)
628 631 def simulator_development(self):
629 632 self.simulator(False)
630   -
  633 +
  634 +
631 635 # Simulation (by default, with ALL simulators and ALL celery workers)
632 636 def simulator(self, TOTAL=True):
633 637 self.changeDirectory("src")
... ... @@ -639,7 +643,10 @@ class Pyros(AManager):
639 643 if TOTAL: self.reset_database_sim()
640 644 self.migrate()
641 645 self.loaddata()
642   - # Launch Django web server
  646 +
  647 + #
  648 + # 1) Launch Django web server
  649 + #
643 650 self.server()
644 651 self.sleep(2)
645 652 self.printFullTerm(Colors.WARNING, "SUMMARY")
... ... @@ -651,35 +658,39 @@ class Pyros(AManager):
651 658 self.printColor(Colors.GREEN, "If the simulation isn't correctly killed, please switch the variable")
652 659 self.printColor(Colors.GREEN, "CELERY_TEST in src/pyros/settings.py to False")
653 660 self.printFullTerm(Colors.WARNING, "SUMMARY")
654   - self.changeDirectory("simulators/config")
655   - self.printColor(Colors.BOLD, "Existing simulations : ", eol='')
656   - sys.stdout.flush()
657   - if (self.system == "Windows") : self.execProcessSilent("dir /B conf*.json")
658   - else: self.execProcessSilent("ls conf*.json")
659   - self.changeDirectory("..")
660   - conf = self.askQuestion("Which simulation do you want to use", default="conf.json")
661   - self.changeDirectory("..")
  661 +
  662 + #
  663 + # 2) Start Celery workers
  664 + #
662 665 self.celery_on(TOTAL)
663 666 self.sleep(3)
664   -
  667 +
  668 + #
  669 + # 3) Start simulator(s) :
  670 + #
  671 + self.sims_launch(TOTAL)
  672 +
  673 + '''
665 674 # (DEFAULT NORMAL) TOTAL RUN
  675 + # - Launch ALL simulators
666 676 if TOTAL:
667   - #self.celery_on()
668   - #self.sleep(3)
669 677 # Launch ALL simulators and WAIT until finished:
670   - self.sims_launch(conf)
671   -
  678 + #self.sims_launch(conf)
  679 + self.sims_launch(TOTAL)
  680 +
672 681 # (DEV) PARTIAL RUN
  682 + # - Launch only USER simulator
673 683 else:
674   - '''
675   - self.singleWorker("scheduling")
676   - self.singleWorker("majordome")
677   - self.singleWorker("execute_plan_vis")
678   - self.singleWorker("execute_plan_nir")
679   - self.singleWorker("create_calibrations")
680   - self.singleWorker("analysis")
681   - self.sleep(3)
682   - '''
  684 + # EP: bugfix: inutile si TOTAL, car refait dans sims_launch !!!
  685 + self.changeDirectory("simulators/config")
  686 + self.printColor(Colors.BOLD, "Existing simulations : ", eol='')
  687 + sys.stdout.flush()
  688 + if (self.system == "Windows") : self.execProcessSilent("dir /B conf*.json")
  689 + else: self.execProcessSilent("ls conf*.json")
  690 + self.changeDirectory("..")
  691 + conf = self.askQuestion("Which simulation do you want to use", default="conf.json")
  692 + self.changeDirectory("..")
  693 +
683 694 procs = []
684 695 self.changeDirectory("simulators")
685 696 self.changeDirectory("user")
... ... @@ -689,9 +700,10 @@ class Pyros(AManager):
689 700 for p in procs:
690 701 p.wait()
691 702 self.changeDirectory("..")
692   -
  703 + '''
  704 +
693 705 # When simulators are finished:
694   - self.kill_simulation()
  706 + #self.kill_simulation()
695 707 return 0
696 708  
697 709  
... ... @@ -725,9 +737,11 @@ class Pyros(AManager):
725 737 self.printFullTerm(Colors.GREEN, "simulation ended")
726 738 return 0
727 739  
  740 +
728 741 # Simulators only
729   - def sims_launch(self, conf=""):
730   - procs = []
  742 + def sims_launch(self, TOTAL=True, conf=""):
  743 +
  744 + # Read simulators scenario file
731 745 self.changeDirectory("simulators/config")
732 746 if (conf == ""):
733 747 self.printColor(Colors.BOLD, "Existing simulations : ", eol='')
... ... @@ -740,32 +754,47 @@ class Pyros(AManager):
740 754 if not os.path.isfile(conf):
741 755 self.printColor(Colors.FAIL, "The simulation file " + conf + " does not exist")
742 756 return 1
743   - self.changeDirectory("..")
744   - self.changeDirectory("dome")
745   - procs.append(self.execProcessFromVenvAsync(self.venv_bin + " domeSimulator.py " + conf))
  757 +
  758 + procs = []
  759 +
  760 + # Launch the User simulator:
746 761 self.changeDirectory("..")
747 762 self.changeDirectory("user")
748 763 procs.append(self.execProcessFromVenvAsync(self.venv_bin + " userSimulator.py " + conf))
749   - self.changeDirectory("..")
750   - self.changeDirectory("alert")
751   - procs.append(self.execProcessFromVenvAsync(self.venv_bin + " alertSimulator.py " + conf))
752   - self.changeDirectory("..")
753   - self.changeDirectory("plc")
754   - procs.append(self.execProcessFromVenvAsync(self.venv_bin + " plcSimulator.py " + conf))
755   - self.changeDirectory("..")
756   - self.changeDirectory("camera")
757   - procs.append(self.execProcessFromVenvAsync(self.venv_bin + " cameraVISSimulator.py " + conf))
758   - procs.append(self.execProcessFromVenvAsync(self.venv_bin + " cameraNIRSimulator.py " + conf))
759   - self.changeDirectory("..")
760   - self.changeDirectory("telescope")
761   - procs.append(self.execProcessFromVenvAsync(self.venv_bin + " telescopeSimulator.py " + conf))
  764 +
  765 + # (if TOTAL) Launch all other simulators
  766 + if TOTAL:
  767 +
  768 + self.changeDirectory("..")
  769 + self.changeDirectory("dome")
  770 + procs.append(self.execProcessFromVenvAsync(self.venv_bin + " domeSimulator.py " + conf))
  771 +
  772 + self.changeDirectory("..")
  773 + self.changeDirectory("alert")
  774 + procs.append(self.execProcessFromVenvAsync(self.venv_bin + " alertSimulator.py " + conf))
  775 +
  776 + self.changeDirectory("..")
  777 + self.changeDirectory("plc")
  778 + procs.append(self.execProcessFromVenvAsync(self.venv_bin + " plcSimulator.py " + conf))
  779 +
  780 + self.changeDirectory("..")
  781 + self.changeDirectory("camera")
  782 + procs.append(self.execProcessFromVenvAsync(self.venv_bin + " cameraVISSimulator.py " + conf))
  783 + procs.append(self.execProcessFromVenvAsync(self.venv_bin + " cameraNIRSimulator.py " + conf))
  784 +
  785 + self.changeDirectory("..")
  786 + self.changeDirectory("telescope")
  787 + procs.append(self.execProcessFromVenvAsync(self.venv_bin + " telescopeSimulator.py " + conf))
  788 +
  789 + # Get back to project root folder and wait for end of simulators :
762 790 self.changeDirectory("..")
763 791 self.changeDirectory("..")
764 792 for p in procs:
765 793 p.wait()
766   - #self.kill_simulation()
  794 + self.kill_simulation()
767 795 return 0
768 796  
  797 +
769 798 def singleWorker(self, worker):
770 799 self.changeDirectory("src")
771 800 self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q "+ worker +"_q -n pyros@"+worker+" -c 1")
... ...
src/monitoring/tasks.py
  Diff comments   View file @d1e6236
... ... @@ -23,7 +23,8 @@ DEBUG_FILE = True
23 23  
24 24 # Set this to False if you want to test this Environment Monitoring agent ALONE, without using Celery (and RabbitMQ)
25 25 # Set this to True if you want to test this Environment Monitoring agent with Celery (and RabbitMQ)
26   -USE_CELERY = False
  26 +#USE_CELERY = False
  27 +USE_CELERY = True
27 28  
28 29  
29 30 '''
... ...
src/pyros/__init__.py
  Diff comments   View file @d1e6236
1 1 from __future__ import absolute_import, unicode_literals
2 2  
  3 +#USE_CELERY=False
  4 +USE_CELERY=True
  5 +
3 6 # This will make sure the app is always imported when
4 7 # Django starts so that shared_task will use this app.
5 8 from .celery import app as celery_app
6 9  
7   -# EP : normalement, il faudrait a jouter cette ligne, à tester
8   -#__all__ = ['celery_app']
9   -
10 10 # EP : normalement aussi, toutes ces autres lignes sont inutiles, à tester
11   -
12 11 from celery.signals import worker_ready
13   -
14 12 from django.conf import settings
15 13  
  14 +# EP : normalement, il faudrait a jouter cette ligne, à tester
  15 +#__all__ = ['celery_app']
  16 +
  17 +
16 18 # (EP) To be deactivated when testing only 1 app in isolation (like Monitoring, or Majordome, or AlertManager...) :
17 19 # Uncomment this block when running Pyros with all modules at the same time
18   -'''
19   -@worker_ready.connect
20   -def start_permanent_tasks(signal, sender):
21   - import monitoring.tasks
22   - import alert_manager.tasks
23   - import majordome.tasks
  20 +if USE_CELERY:
  21 + @worker_ready.connect
  22 + def start_permanent_tasks(signal, sender):
  23 + import monitoring.tasks
  24 + import alert_manager.tasks
  25 + import majordome.tasks
24 26  
25   - if sender.hostname == "pyros@monitoring":
26   - monitoring.tasks.Monitoring.delay()
27   - elif sender.hostname == "pyros@majordome":
28   - majordome.tasks.Majordome.delay()
29   - elif sender.hostname == "pyros@alert_listener":
30   - alert_manager.tasks.AlertListener.delay()
31   -'''
  27 + if sender.hostname == "pyros@monitoring":
  28 + monitoring.tasks.Monitoring.delay()
  29 + elif sender.hostname == "pyros@majordome":
  30 + majordome.tasks.Majordome.delay()
  31 + elif sender.hostname == "pyros@alert_listener":
  32 + alert_manager.tasks.AlertListener.delay()
... ...
src/pyros/settings.py
  Diff comments   View file @d1e6236
... ... @@ -197,10 +197,11 @@ USE_L10N = True
197 197 # If you set this to False, Django will not use timezone-aware datetimes.
198 198 # If true => "updated" fields in weatherwatch or sitewatch will be saved as UTC time
199 199 # If false => "updated" fields in weatherwatch or sitewatch will be saved as UTC+1 time (French time)
200   -# Necessary for "pyros test" :
201   -#USE_TZ = True
  200 +# (Was) Necessary for "pyros test" (no more necessary now because fixtures dates have been converted to naive format without time zone) :
  201 +# Still necessary for "pyros.py simulator" (because of some fixtures that still use timezone-aware datetimes)
  202 +USE_TZ = True
202 203 # Necessary for ENV monitoring :
203   -USE_TZ = False
  204 +#USE_TZ = False
204 205  
205 206 # To find the media files {{ MEDIA_URL }}
206 207 MEDIA_URL = '/public/static/media/'
... ...
src/scheduler/Scheduler.py
  Diff comments   View file @d1e6236
... ... @@ -101,7 +101,7 @@ class Scheduler(IntervalManagement):
101 101 return 0
102 102  
103 103 '''
104   - Default entry point
  104 + Default entry point (called from scheduler/tasks.py/scheduling/run())
105 105 '''
106 106 def makeSchedule(self) -> Schedule:
107 107 global SIMULATION
... ...