PYROS-IRAP / pyros

Commit f48ec0d3b6ecab3e45537174894d35d6c027e332

Authored by Alain Klotz
2 parents 7200894b f05b7a2d
Exists in dev

Merge branch 'dev' of https://gitlab.irap.omp.eu/epallier/pyros into dev

Showing 21 changed files with 196 additions and 615 deletions   Show diff stats
Inline Side-by-side
README.md
  Diff comments   View file @f48ec0d
... ... @@ -67,13 +67,14 @@ This software has been tested and validated with the following configurations :
67 67 --------------------------------------------------------------------------------------------
68 68 ## LAST VERSION
69 69  
70   -Date: 15/10/2018
  70 +Date: 17/01/2019
71 71  
72   -By: E. Pallier
  72 +Author: E. Pallier
73 73  
74   -VERSION: 20181015.0.011.2 (bugfix)
  74 +VERSION: 20190117.0.000.0
75 75  
76   -- Bugfix majordome_test.py
  76 +Comment:
  77 + NO MORE CELERY (removed from everywhere)
77 78  
78 79 --------------------------------------------------------------------------------------------
79 80 - TECHNICAL DOC: tinyurl/pyros-colibri
... ... @@ -92,6 +93,9 @@ VERSION: 20181015.0.011.2 (bugfix)
92 93  
93 94 Full list of commits: https://gitlab.irap.omp.eu/epallier/pyros/commits/master
94 95  
  96 +**17/01/19 : Version 20190117.0.000.0 (EP)**
  97 +- NO MORE CELERY (removed from everywhere)
  98 +
95 99 **10/10/18 : Version 2018.0.011 (E. Pallier)**
96 100 - NEW versionning plan : YYYYMMDD.M.mmm.b
97 101 - YYYYMMDD = Day (UTC)
... ...
install/REQUIREMENTS.txt
  Diff comments   View file @f48ec0d
1   -amqp==1.4.9
2   -amqplib==1.0.2
3   -anyjson==0.3.3
4   -billiard==3.3.0.23
5   -requests==2.18.4
6   -celery==3.1.25
7   -DateTime==4.2
8   -Django==2.0.5
9   -django-admin-tools==0.8.1
10   -django-bootstrap3==10.0.1
11   -django-celery==3.2.2
12   -django-debug-toolbar==1.9.1
13   -django-extensions==2.0.7
14   -django-suit==0.2.26
15   -django-test-without-migrations==0.6
16   -gunicorn==19.8.1
17   -iso8601==0.1.12
18   -jdcal==1.4
19   -kombu==3.0.37
20   -lxml==3.8.0
21   -mysqlclient==1.3.12
22   -#pluggy==0.6.0
23   -pluggy>=0.7
24   -py==1.5.3
25   -pytz==2018.4
26   -six==1.11.0
27   -sqlparse==0.2.4
28   -Twisted==18.4.0
29   -voevent-parse==1.0.2
30   -zope.interface==4.5.0
31   -flower==0.9.2
  1 +amqp==1.4.9
  2 +amqplib==1.0.2
  3 +anyjson==0.3.3
  4 +billiard==3.3.0.23
  5 +requests==2.18.4
  6 +DateTime==4.2
  7 +Django==2.0.5
  8 +django-admin-tools==0.8.1
  9 +django-bootstrap3==10.0.1
  10 +django-debug-toolbar==1.9.1
  11 +django-extensions==2.0.7
  12 +django-suit==0.2.26
  13 +django-test-without-migrations==0.6
  14 +gunicorn==19.8.1
  15 +iso8601==0.1.12
  16 +jdcal==1.4
  17 +kombu==3.0.37
  18 +lxml==3.8.0
  19 +mysqlclient==1.3.12
  20 +#pluggy==0.6.0
  21 +pluggy>=0.7
  22 +py==1.5.3
  23 +pytz==2018.4
  24 +six==1.11.0
  25 +sqlparse==0.2.4
  26 +Twisted==18.4.0
  27 +voevent-parse==1.0.2
  28 +zope.interface==4.5.0
  29 +flower==0.9.2
... ...
install/REQUIREMENTS_WINDOWS.txt
  Diff comments   View file @f48ec0d
1   -amqp==1.4.9
2   -amqplib==1.0.2
3   -anyjson==0.3.3
4   -billiard==3.3.0.23
5   -requests==2.18.4
6   -celery==3.1.25
7   -DateTime==4.2
8   -Django==2.0.5
9   -django-admin-tools==0.8.1
10   -django-bootstrap3==10.0.1
11   -django-celery==3.2.2
12   -django-debug-toolbar==1.9.1
13   -django-extensions==2.0.7
14   -django-suit==0.2.26
15   -django-test-without-migrations==0.6
16   -gunicorn==19.4.5
17   -iso8601==0.1.12
18   -jdcal==1.4
19   -kombu==3.0.37
20   -mysqlclient==1.3.12
21   -pluggy==0.6.0
22   -py==1.5.3
23   -pytz==2018.4
24   -six==1.11.0
25   -sqlparse==0.2.4
26   -Twisted==16.0.0
27   -zope.interface==4.5.0
  1 +amqp==1.4.9
  2 +amqplib==1.0.2
  3 +anyjson==0.3.3
  4 +billiard==3.3.0.23
  5 +requests==2.18.4
  6 +DateTime==4.2
  7 +Django==2.0.5
  8 +django-admin-tools==0.8.1
  9 +django-bootstrap3==10.0.1
  10 +django-debug-toolbar==1.9.1
  11 +django-extensions==2.0.7
  12 +django-suit==0.2.26
  13 +django-test-without-migrations==0.6
  14 +gunicorn==19.4.5
  15 +iso8601==0.1.12
  16 +jdcal==1.4
  17 +kombu==3.0.37
  18 +mysqlclient==1.3.12
  19 +pluggy==0.6.0
  20 +py==1.5.3
  21 +pytz==2018.4
  22 +six==1.11.0
  23 +sqlparse==0.2.4
  24 +Twisted==16.0.0
  25 +zope.interface==4.5.0
... ...
pyros.py
  Diff comments   View file @f48ec0d
... ... @@ -9,10 +9,6 @@ import argparse
9 9 import time
10 10 import signal
11 11  
12   -# For USE_CELERY global variable
13   -#from src.pyros import settings
14   -USE_CELERY = False
15   -#USE_CELERY = True
16 12  
17 13 DEBUG = True
18 14 SIMULATOR_CONFIG_FILE = "conf.json" #default config file
... ... @@ -54,28 +50,27 @@ class Utils:
54 50 print("Current directory : " + str(os.getcwd()))
55 51 return 0
56 52  
57   - def replacePatternInFile(self, pattern, replace, file_path):
  53 + def replacePatternInFile(self, pattern, replacement, file_path):
58 54 #try:
59 55 with fileinput.FileInput(file_path, inplace=True, backup='.bak') as file:
60 56 for line in file:
61   - print(line.replace(pattern, replace), end='')
  57 + print(line.replace(pattern, replacement), end='')
62 58 '''
63 59 except:
64 60 return 1
65 61 return 0
66 62 '''
67 63 # Now, check that replacement has been done or else ERROR !!!
68   - #pattern = "CELERY_TEST = "+str(set_it)
69 64 FOUND=False
70 65 #with fileinput.FileInput(file_path) as file:
71 66 with open(file_path) as file:
72 67 for line in file:
73   - #if replace in line:
74   - if line.startswith(replace):
  68 + #if replacement in line:
  69 + if line.startswith(replacement):
75 70 FOUND = True
76 71 break
77 72 #if FOUND: print("pattern "+pattern+" found in file "+file_path)
78   - if not FOUND: raise(Exception("pattern "+replace+" not found in file "+file_path))
  73 + if not FOUND: raise(Exception("pattern "+replacement+" not found in file "+file_path))
79 74  
80 75  
81 76 def printColor(self, color, message, file=sys.stdout, eol=os.linesep, forced=False):
... ... @@ -113,7 +108,6 @@ class AManager(Utils):
113 108 python_version = sys.version_info
114 109  
115 110 bin_dir = ""
116   - celery = "celery"
117 111 venv_pip = "pip"
118 112 venv_bin = "python"
119 113 wait = True
... ... @@ -141,34 +135,29 @@ class AManager(Utils):
141 135 self.bin_dir = "Scripts"
142 136 self.bin_name = "python.exe"
143 137 self.pip_name = "pip.exe"
144   - self.celery = "celery.exe"
145 138 else:
146 139 self.bin_dir = "bin"
147 140 self.bin_name = "python"
148 141 self.pip_name = "pip"
149   - self.celery = "celery"
150 142 self.venv_pip = self.path_dir_file + os.sep + config["path"] + os.sep + config["env"] + os.sep + self.bin_dir + os.sep + self.pip_name
151 143 self.venv_bin = self.path_dir_file + os.sep + config["path"] + os.sep + config["env"] + os.sep + self.bin_dir + os.sep + self.bin_name
152   - self.venv_cel = self.path_dir_file + os.sep + config["path"] + os.sep + config["env"] + os.sep + self.bin_dir + os.sep + self.celery
153 144  
154 145 def help(self):
155 146 print("This function must be implemented")
156 147 raise(NotImplementedError("Function not implemented"))
157 148  
158 149  
159   - def set_celery_test_and_simulator_modes_to(self, set_it:bool, simulator:bool=True):
  150 + def set_simulator_mode_to(self, set_it:bool):
160 151 file_path = "pyros/settings.py"
161 152 '''
162   - self.replacePatternInFile("CELERY_TEST = False", "CELERY_TEST = True", file_path)
163 153 if simulator: self.replacePatternInFile("SIMULATOR = False", "SIMULATOR = True", file_path)
164 154 '''
165   - self.replacePatternInFile("CELERY_TEST = "+str(not set_it), "CELERY_TEST = "+str(set_it), file_path)
166   - if simulator: self.replacePatternInFile("SIMULATOR = "+str(not set_it), "SIMULATOR = "+str(set_it), file_path)
  155 + self.replacePatternInFile("SIMULATOR = "+str(not set_it), "SIMULATOR = "+str(set_it), file_path)
167 156  
168 157  
169 158 def signal_handler(self, signal, frame):
170 159 self.printFullTerm(Colors.WARNING, "Ctrl-c catched")
171   - self.set_celery_test_and_simulator_modes_to(False)
  160 + self.set_simulator_mode_to(False)
172 161 for p in self.subproc:
173 162 proc, name = p
174 163 self.printColor(Colors.BLUE, "Killing process " + str(name))
... ... @@ -448,12 +437,10 @@ class Pyros(AManager):
448 437 return self.execProcess("rm logs/*.log")
449 438  
450 439 def test(self):
451   - ##self.singleWorker("scheduling")
452 440 self.changeDirectory("src")
453 441 ##self.execProcessFromVenv(self.venv_bin + " manage.py test")
454 442 self.execProcessFromVenvAsync(self.venv_bin + " manage.py test")
455 443 self.changeDirectory("..")
456   - ##self.stop_workers()
457 444 return 0
458 445  
459 446 def migrate(self):
... ... @@ -487,7 +474,7 @@ class Pyros(AManager):
487 474  
488 475 def reset_config(self):
489 476 self.changeDirectory("src")
490   - self.set_celery_test_and_simulator_modes_to(False)
  477 + self.set_simulator_mode_to(False)
491 478 self.addExecuted(self.current_command, "reset configuration")
492 479 self.changeDirectory("..")
493 480 return 0
... ... @@ -514,50 +501,6 @@ class Pyros(AManager):
514 501 return 0
515 502  
516 503  
517   - def celery_on(self, TOTAL=True):
518   - self.changeDirectory("src")
519   -
520   - if TOTAL: self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q alert_listener_q -n pyros@alert_listener -c 1")
521   - if TOTAL: self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q monitoring_q -n pyros@monitoring -c 1")
522   -
523   - #self.singleWorker("majordome")
524   - self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q majordome_q -n pyros@majordome -c 1")
525   -
526   - #self.singleWorker("scheduling")
527   - self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q scheduling_q --purge -n pyros@scheduling -c 1")
528   -
529   - #self.singleWorker("execute_plan_vis")
530   - self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q execute_plan_vis_q --purge -n pyros@execute_plan_vis -c 1")
531   -
532   - if TOTAL: self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q night_calibrations_q --purge -n pyros@night_calibrations -c 1")
533   -
534   - #self.singleWorker("execute_plan_nir")
535   - self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q execute_plan_nir_q --purge -n pyros@execute_plan_nir -c 1")
536   -
537   - #self.singleWorker("create_calibrations")
538   - self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q create_calibrations_q --purge -n pyros@create_calibrations -c 1")
539   -
540   - #self.singleWorker("analysis")
541   - self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q analysis_q --purge -n pyros@analysis -c 1")
542   -
543   - self.changeDirectory("..")
544   - return 0
545   -
546   - # ex start()
547   - def start_workers(self):
548   - self.stop_workers()
549   - self.celery_on()
550   - return 0
551   -
552   - # ex stop()
553   - def stop_workers(self):
554   - if (self.system == "Windows"):
555   - self.execProcessAsync("taskkill /f /im celery.exe")
556   - self.execProcessAsync("taskkill /f /im python.exe")
557   - else:
558   - self.execProcessAsync("ps aux | grep \"celery worker\" | awk '{print $2}' | xargs kill -9")
559   - return 0
560   -
561 504 def init_database(self):
562 505 self.makemigrations()
563 506 self.migrate()
... ... @@ -596,15 +539,6 @@ class Pyros(AManager):
596 539 what="all"
597 540 '''
598 541  
599   - ''' OLD CODE
600   - self.stop_workers()
601   - #self.celery_on()
602   - # Start the 3 main agents (via their celery worker) :
603   - self.startAgent("monitoring")
604   - self.startAgent("majordome")
605   - self.startAgent("alert_listener")
606   - '''
607   -
608 542 # Go into src/
609 543 self.changeDirectory("src")
610 544 #print("Current directory : " + str(os.getcwd()))
... ... @@ -655,17 +589,10 @@ class Pyros(AManager):
655 589  
656 590  
657 591  
658   - def startAgent(self, agent_name:str):
659   - # Start Celery Worker
660   - self.singleWorker(agent_name)
661   - time.sleep(3)
662   - # Put the agent task run() into the Worker queue
663   -
664   -
665 592 # Reset the database content
666 593 def reset_database_sim(self):
667 594 self.changeDirectory("src")
668   - self.set_celery_test_and_simulator_modes_to(True, False)
  595 + self.set_simulator_mode_to(True, False)
669 596 '''
670 597 Supprime toutes les données de la base de données
671 598 et réexécute tout gestionnaire de post-synchronisation.
... ... @@ -673,11 +600,11 @@ class Pyros(AManager):
673 600 '''
674 601 #TODO: remplacer par manage.py --noinput flush pour eviter le "echo yes"...
675 602 self.execProcess( self.venv_bin + " manage.py flush --noinput")
676   - self.set_celery_test_and_simulator_modes_to(True, False)
  603 + self.set_simulator_mode_to(True, False)
677 604 self.changeDirectory("..")
678 605  
679 606  
680   - # Simulation for the scheduler ONLY (and only some celery workers, not all of them)
  607 + # Simulation for the scheduler ONLY
681 608 def simulator_development(self):
682 609 self.simulator(False)
683 610  
... ... @@ -691,12 +618,11 @@ class Pyros(AManager):
691 618 delete from request;
692 619 '''
693 620  
694   - # Simulation (by default, with ALL simulators and ALL celery workers)
  621 + # Simulation (by default, with ALL simulators)
695 622 def simulator(self, TOTAL=True):
696 623 self.changeDirectory("src")
697 624  
698   - # Set CELERY_TEST mode ON
699   - self.replacePatternInFile("CELERY_TEST = False", "CELERY_TEST = True", "pyros/settings.py")
  625 + # Set SIMULATOR mode ON
700 626 self.replacePatternInFile("SIMULATOR = False", "SIMULATOR = True", "pyros/settings.py")
701 627  
702 628 # 0) Reset the database (Empty the database from any data)
... ... @@ -714,7 +640,7 @@ class Pyros(AManager):
714 640 self.loaddata()
715 641  
716 642 #
717   - # 1) Launch Django web server
  643 + # 1) Launch web server
718 644 #
719 645 self.server()
720 646 self.sleep(2)
... ... @@ -727,18 +653,11 @@ class Pyros(AManager):
727 653 self.printColor(Colors.GREEN, "If you want to shutdown the simulation, please run :")
728 654 self.printColor(Colors.GREEN, "CTRL-C or pyros.py kill_simulation")
729 655 self.printColor(Colors.GREEN, "If the simulation isn't correctly killed, please switch the variable")
730   - self.printColor(Colors.GREEN, "CELERY_TEST in src/pyros/settings.py to False")
  656 + self.printColor(Colors.GREEN, "SIMULATOR in src/pyros/settings.py to False")
731 657 self.printFullTerm(Colors.WARNING, "SUMMARY")
732 658  
733 659 #
734   - # 2) (if USE_CELERY) Start Celery workers
735   - #
736   - if USE_CELERY:
737   - self.celery_on(TOTAL)
738   - self.sleep(3)
739   -
740   - #
741   - # 3) Start simulator(s) :
  660 + # 2) Start simulator(s) :
742 661 #
743 662 self.sims_launch(TOTAL)
744 663  
... ... @@ -774,20 +693,6 @@ class Pyros(AManager):
774 693 self.changeDirectory("..")
775 694 '''
776 695  
777   - '''
778   - #
779   - # 2) (if not USE_CELERY) Start Agents
780   - #
781   - if TOTAL and not USE_CELERY :
782   - #pass
783   - # Start Environment Monitoring Agent
784   - # Start Majordome Agent
785   - # Start Alert Manager Agent
786   - self.start_agents()
787   - # Needed ?
788   - self.sleep(3)
789   - '''
790   -
791 696 # When simulators are finished:
792 697 #self.kill_simulation()
793 698 return 0
... ... @@ -799,8 +704,8 @@ class Pyros(AManager):
799 704 def test_majordome(self):
800 705 self.changeDirectory("src")
801 706  
802   - # Set CELERY_TEST mode ON
803   - self.set_celery_test_and_simulator_modes_to(True)
  707 + # Set SIMULATOR mode ON
  708 + self.set_simulator_mode_to(True)
804 709  
805 710 # 0) Reset the database (Empty the database from any data)
806 711 self.changeDirectory("..")
... ... @@ -824,19 +729,11 @@ class Pyros(AManager):
824 729 self.printColor(Colors.GREEN, "If you want to shutdown the simulation, please run :")
825 730 self.printColor(Colors.GREEN, "CTRL-C or pyros.py kill_simulation")
826 731 self.printColor(Colors.GREEN, "If the simulation isn't correctly killed, please switch the variable")
827   - self.printColor(Colors.GREEN, "CELERY_TEST in src/pyros/settings.py to False")
  732 + self.printColor(Colors.GREEN, "SIMULATOR in src/pyros/settings.py to False")
828 733 self.printFullTerm(Colors.WARNING, "SUMMARY")
829 734  
830   - # 2) (if USE_CELERY) Start Celery workers
831 735 #
832   - '''
833   - if USE_CELERY:
834   - self.celery_on(TOTAL)
835   - self.sleep(3)
836   - '''
837   -
838   - #
839   - # 3) Start simulator(s) :
  736 + # 2) Start simulator(s) :
840 737 #
841 738 #self.sims_launch(True)
842 739 agent='majordome'
... ... @@ -852,7 +749,7 @@ class Pyros(AManager):
852 749 # When simulators are finished:
853 750 #self.kill_simulation()
854 751 #self.changeDirectory("src")
855   - self.set_celery_test_and_simulator_modes_to(False)
  752 + self.set_simulator_mode_to(False)
856 753 self.changeDirectory("..")
857 754 self.execProcessAsync("ps aux | grep \"start_agent_majordome.py\" | awk '{ print $2 }' | xargs kill")
858 755  
... ... @@ -870,7 +767,7 @@ class Pyros(AManager):
870 767  
871 768 def kill_simulation(self):
872 769 self.changeDirectory("src")
873   - self.set_celery_test_and_simulator_modes_to(False)
  770 + self.set_simulator_mode_to(False)
874 771 self.changeDirectory("..")
875 772 if (self.system == "Windows"):
876 773 self.execProcessAsync("taskkill /f /im python.exe")
... ... @@ -884,12 +781,12 @@ class Pyros(AManager):
884 781  
885 782 self.kill_server()
886 783  
887   - # (if not using celery) Kill all agents :
888   - if not USE_CELERY:
889   - self.execProcessAsync("ps aux | grep \"start_agent_alert_manager.py\" | awk '{ print $2 }' | xargs kill")
890   - self.execProcessAsync("ps aux | grep \"start_agent_monitoring.py\" | awk '{ print $2 }' | xargs kill")
891   - self.execProcessAsync("ps aux | grep \"start_agent_majordome.py\" | awk '{ print $2 }' | xargs kill")
  784 + # Kill all agents :
  785 + self.execProcessAsync("ps aux | grep \"start_agent_alert_manager.py\" | awk '{ print $2 }' | xargs kill")
  786 + self.execProcessAsync("ps aux | grep \"start_agent_monitoring.py\" | awk '{ print $2 }' | xargs kill")
  787 + self.execProcessAsync("ps aux | grep \"start_agent_majordome.py\" | awk '{ print $2 }' | xargs kill")
892 788 #self.execProcessAsync("ps aux | grep \"start_agent_majordome.py\" | awk '{ print $2 }' | xargs kill")
  789 +
893 790 # Kill all simulators :
894 791 self.execProcessAsync("ps aux | grep \" domeSimulator.py\" | awk '{ print $2 }' | xargs kill")
895 792 self.execProcessAsync("ps aux | grep \" userSimulator.py\" | awk '{ print $2 }' | xargs kill")
... ... @@ -899,7 +796,6 @@ class Pyros(AManager):
899 796 self.execProcessAsync("ps aux | grep \" cameraNIRSimulator.py\" | awk '{ print $2 }' | xargs kill")
900 797 self.execProcessAsync("ps aux | grep \" cameraVISSimulator.py\" | awk '{ print $2 }' | xargs kill")
901 798 self.changeDirectory("..")
902   - if USE_CELERY: self.stop_workers()
903 799  
904 800 self.printFullTerm(Colors.GREEN, "simulation ended")
905 801 return 0
... ... @@ -1001,8 +897,8 @@ class Pyros(AManager):
1001 897 # Get back to project root folder
1002 898 self.changeDirectory("..")
1003 899  
1004   - # (if not celery) Launch agents (env monitor, major, alert mgr)
1005   - if not USE_CELERY: self.start_agents()
  900 + # Launch agents (env monitor, major, alert mgr)
  901 + self.start_agents()
1006 902  
1007 903 # Wait for end of simulators :
1008 904 for p in procs: p.wait()
... ... @@ -1012,12 +908,6 @@ class Pyros(AManager):
1012 908 return 0
1013 909  
1014 910  
1015   - def singleWorker(self, worker):
1016   - self.changeDirectory("src")
1017   - self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q "+ worker +"_q -n pyros@"+worker+" -c 1")
1018   - self.changeDirectory("..")
1019   - return 0
1020   -
1021 911 def mysql_on(self):
1022 912 self.changeDirectory("src")
1023 913 self.replacePatternInFile("MYSQL = False", "MYSQL = True", "pyros/settings.py")
... ... @@ -1050,7 +940,6 @@ class Pyros(AManager):
1050 940 "unittest": self.unittest,
1051 941 "reset_config": self.reset_config,
1052 942 "test_all": self.test_all,
1053   - "celery_on": self.celery_on,
1054 943 "reset_database_sim": self.reset_database_sim,
1055 944 "init_database": self.init_database,
1056 945 "kill_server": self.kill_server,
... ... @@ -1065,10 +954,6 @@ class Pyros(AManager):
1065 954 "start_agent_alertmanager": self.start_agent_alertmanager,
1066 955 "test_majordome": self.test_majordome,
1067 956  
1068   - # ex start
1069   - "start_workers": self.start_workers,
1070   - # ex stop
1071   - "stop_workers": self.stop_workers,
1072 957 "simulator": self.simulator,
1073 958 "kill_simulation": self.kill_simulation,
1074 959 "sims_launch": self.sims_launch,
... ... @@ -1092,14 +977,11 @@ class Pyros(AManager):
1092 977 "updatedb": "Update the database",
1093 978 "kill_server": "Kill the web server on port 8000",
1094 979 "init_database": "Create a standard context for pyros in db",
1095   - "unittest": "Runs the tests that don't need celery",
  980 + "unittest": "Runs the unit tests",
1096 981 "test_all": "Run all the existing tests (this command needs to be updated when tests are added in the project",
1097   - "celery_on": "Starts celery workers",
1098   - "start_workers": "Starts the celery workers (then the web server ???)",
1099   - "stop_workers": "Stops the celery workers",
1100 982 "simulator": "Launch a simulation",
1101 983 "simulator_development": "Simulation for the scheduler only",
1102   - "kill_simulation": "kill the simulators / celery workers / web server",
  984 + "kill_simulation": "kill the simulators & web server",
1103 985 "sims_launch": "Launch only the simulators",
1104 986 }
1105 987  
... ...
src/alert_manager/tasks.py
  Diff comments   View file @f48ec0d
1 1 from __future__ import absolute_import
2   -from celery.task import Task
3 2 from alert_manager.StrategyBuilder import StrategyBuilder
4 3 from common.models import *
5 4 from django.conf import settings
... ... @@ -23,7 +22,7 @@ DEBUG_FILE = False
23 22 '''
24 23  
25 24  
26   -class AlertListener(Task):
  25 +class AlertListener():
27 26 old_files = []
28 27  
29 28 # TODO check monitoring and majordome status ?
... ...
src/alert_manager/tests.py
  Diff comments   View file @f48ec0d
... ... @@ -78,51 +78,3 @@ class TestStrategyChange(TestCase):
78 78 new_alert = Alert.objects.exclude(id=self.alert.id)[0]
79 79 self.assertEqual(new_alert.strategyobs.id, self.strat2.id, "The new alert should have the 'strat2' strategy")
80 80  
81   -# OBSOLETE
82   -# class AlertListenerTestsCelery(TestCase):
83   -# '''
84   -# IMPORTANT : As soon as you do a DB request in a test, the test DB will no longer be synchronized with the DB used by celery
85   -# I have no idea why (Paul), but any call to the DB must be done in setup or after all celery actions are finished
86   -# '''
87   -#
88   -# def test_alert_reception(self):
89   -# '''
90   -# Must be launched with scripts/celery_test.sh
91   -# Copy a VOEvent file in the events_received directory to start the workflow
92   -# Tests if the alert and children are well created
93   -# '''
94   -#
95   -#
96   -# if os.path.isfile(TEST_FILE_PATH):
97   -# os.remove(TEST_FILE_PATH)
98   -# print("================== DELETE FILE ==================")
99   -# time.sleep(3)
100   -#
101   -# print("================== COPY FILE ==================")
102   -# shutil.copyfile(os.path.join(VOEVENTS_TO_SEND_PATH, TEST_FILE),
103   -# TEST_FILE_PATH)
104   -# time.sleep(4)
105   -#
106   -# self.assertEqual(Alert.objects.count(), 1)
107   -# alert = Alert.objects.all()[0]
108   -# self.assertEqual(alert.author, "ivo://nasa.gsfc.tan/gcn")
109   -# self.assertEqual(alert.burst_ra, 74.7412)
110   -# self.assertEqual(alert.burst_dec, 25.3137)
111   -# self.assertEqual(alert.trig_id, 532871)
112   -# self.assertEqual(alert.editor, "61")
113   -# self.assertEqual(alert.pkt_ser_num, 1)
114   -#
115   -# self.assertEqual(Request.objects.count(), 1)
116   -# self.assertEqual(Sequence.objects.count(), 2)
117   -# self.assertEqual(Album.objects.count(), 3)
118   -# self.assertEqual(Plan.objects.count(), 6)
119   -#
120   -#
121   -#
122   -# def test_basic(self):
123   -# '''
124   -# Used to test scripts/celery_test.sh
125   -# Only work if the initial_fixture is on the celery test DB (an only it)
126   -# '''
127   -# Country.objects.create(name="TEEEEST")
128   -# self.assertEqual(Country.objects.count(), 2, "should be 2 countries")
... ...
src/analyzer/tasks.py
  Diff comments   View file @f48ec0d
1 1 from __future__ import absolute_import
2   -from celery.task import Task
3 2 from common.models import *
4 3 from utils.Logger import setupLogger
5 4 from django.conf import settings
... ... @@ -12,7 +11,8 @@ import os
12 11 '''
13 12  
14 13  
15   -class Analysis(Task):
  14 +#class Analysis(Task):
  15 +class Analysis():
16 16 logger = setupLogger("analysis", "analysis")
17 17  
18 18 def log(self, message: str) -> int:
... ...
src/common/models.py
  Diff comments   View file @f48ec0d
... ... @@ -661,6 +661,7 @@ class StrategyObs(models.Model):
661 661 return (str(self.name))
662 662  
663 663  
  664 +#TODO: à virer car utilisé pour Celery (ou bien à utiliser pour les agents)
664 665 class TaskId(models.Model):
665 666 task = models.CharField(max_length=45, blank=True, null=True)
666 667 created = models.DateTimeField(blank=True, null=True, auto_now_add=True)
... ...
src/dashboard/templates/dashboard/reload_weather.html
  Diff comments   View file @f48ec0d
... ... @@ -33,17 +33,26 @@
33 33 padding: 3px 5px 3px 3px;
34 34 margin: 1px 2px 1px 1px;
35 35 }
  36 + div.box {
  37 + /*float:left;*/
  38 + /*display:inline;*/
  39 + font-size: 11pt;
  40 + /*background:#e7e4e4;*/
  41 + border: 1px solid rgba(0, 0, 0, 0.125);
  42 + border-color: #2FA4E7;
  43 + border-radius: 0.25rem;
  44 + padding: 3px 5px 3px 3px;
  45 + margin: 1px 2px 1px 1px;
  46 +}
36 47 </style>
37 48  
38 49 {% endblock %}
39 50  
40 51 {% block content %}
41 52 <div id="page-wrapper" class="row">
42   - <h1 id="none">Weather info</h1>
43   - Maximum of weather info iteration set: {{iteration}} Config 10190108
44   - <div class="row">
45   - <div id="global" class="container-fluid hbox">
46   - <h4>Global</h4>
  53 + <div class="row col-md-3">
  54 + <h3>Weather info</h3>
  55 + <div id="global" class="container-fluid box">
47 56 <table>
48 57 <tr class="alert alert-danger">
49 58 <th scope="row">global_status:&nbsp;</th><td id="global_status">KO</td>
... ... @@ -62,14 +71,17 @@
62 71 </tr>
63 72 </table>
64 73 </div>
65   - <div id="plc" class="container-fluid hbox">
  74 + <div id="plc" class="container-fluid box">
66 75 </div>
67 76 </div>
68   - <div class="container-fluid row">
  77 + <div class="row col-md-8">
69 78 <h3>Weather history </h3>
70   - <div id="refresh" class="container-fluid well">
71   - <div id="humidity_plot"></div>
72   - </div>
  79 + <div id="humidity_plot" class="container-fluid hbox"></div>
  80 + <div id="pressure_plot" class="container-fluid hbox"></div>
  81 + <div id="rain_plot" class="container-fluid hbox"></div>
  82 + <div id="temperature_plot" class="container-fluid hbox"></div>
  83 + <div id="wind_plot" class="container-fluid hbox"></div>
  84 + <div id="wind_dir_plot" class="container-fluid hbox"></div>
73 85 </div>
74 86 </div>
75 87 {% endblock %}
... ...
src/majordome/TaskManager.py
  Diff comments   View file @f48ec0d
1 1 from common.models import *
2 2  
3   -from celery.task.control import revoke
4 3 import time
5 4 from devices import Telescope, CameraVIS, CameraNIR
6 5  
... ... @@ -11,13 +10,14 @@ def delete_pending_tasks(task_names, terminate=False):
11 10 :param task_names: array of tasks names to delete
12 11 '''
13 12  
14   - # TODO : il faudra aller plus loin, et dire aux instruments de s'arrêter
  13 + #TODO : il faudra aller plus loin, et dire aux instruments de s'arrêter
15 14  
16 15 for task_name in task_names:
17 16 tasks = TaskId.objects.filter(task=task_name)
18 17 for task in tasks:
19 18 print("je revoke : ", task.task_id)
20   - revoke(task.task_id, terminate=terminate)
  19 + #TODO:
  20 + ###revoke(task.task_id, terminate=terminate)
21 21 task.delete()
22 22  
23 23 """ The sleep is in case of sequence stopping :
... ...
src/majordome/tasks.py
  Diff comments   View file @f48ec0d
... ... @@ -3,7 +3,6 @@ from __future__ import absolute_import
3 3 import time
4 4 import datetime
5 5  
6   -from celery.task import Task
7 6 #from django.core.exceptions import ObjectDoesNotExist
8 7 #from django.db.models import Q
9 8 from django.shortcuts import get_object_or_404
... ... @@ -48,7 +47,7 @@ log = L.setupLogger(&quot;MajordomeTaskLogger&quot;, &quot;Majordome&quot;)
48 47 '''
49 48  
50 49  
51   -class Majordome(Task):
  50 +class Majordome():
52 51 # (EP) do this so that Majordome can be run from a thread, and called with thread.start():
53 52 #class Majordome(Task, Thread):
54 53  
... ... @@ -84,7 +83,6 @@ class Majordome(Task):
84 83  
85 84  
86 85 '''
87   - OLD //// Function called by celery task
88 86 Behavior:
89 87 Init telescope / cameras
90 88 set night limits
... ... @@ -575,7 +573,7 @@ class Majordome(Task):
575 573 Function called by the main loop to handle the task event (check monitoring and alert_manager)
576 574 '''
577 575 def handleTasks(self):
578   - if not settings.USE_CELERY: return 0
  576 + return 0
579 577  
580 578 self.timers["tasks"] = self.tasks_timer
581 579 if self.monitoring_task is None:
... ... @@ -623,7 +621,7 @@ class Majordome(Task):
623 621 if (settings.DEBUG and DEBUG_FILE):
624 622 log.info("Timer : " + str(timer_name) + " executed")
625 623 # EP added because loop is too quick (without CELERY)
626   - if not settings.USE_CELERY: time.sleep(2)
  624 + time.sleep(2)
627 625 return (0)
628 626  
629 627 '''
... ... @@ -709,12 +707,8 @@ class Majordome(Task):
709 707 self.timers["night_end"] = getNightEnd()
710 708 if (self.isValidStatus(self.status_tel)):
711 709 #observation_manager.tasks.night_calibrations.apply_async()
712   - if settings.USE_CELERY:
713   - print("MJ: call observation_manager WITH CELERY")
714   - observation_manager.tasks.night_calibrations.apply_async()
715   - else:
716   - print("MJ: call observation_manager WITHOUT CELERY")
717   - observation_manager.tasks.night_calibrations().run()
  710 + print("MJ: call observation_manager WITHOUT CELERY")
  711 + observation_manager.tasks.night_calibrations().run()
718 712  
719 713 else:
720 714 self.notifyTelescopeStatus("night_end")
... ... @@ -730,12 +724,8 @@ class Majordome(Task):
730 724 self.vis_camera.open_shutter()
731 725 self.nir_camera.open_shutter()
732 726 #scheduler.tasks.scheduling.apply_async((False, False))
733   - if settings.USE_CELERY:
734   - print("MJ: call schedule WITH CELERY")
735   - scheduler.tasks.scheduling.apply_async((False, False))
736   - else:
737   - print("MJ: call schedule WITHOUT CELERY")
738   - scheduler.tasks.scheduling().run((False, False))
  727 + print("MJ: call schedule WITHOUT CELERY")
  728 + scheduler.tasks.scheduling().run((False, False))
739 729  
740 730 return (0)
741 731  
... ... @@ -805,21 +795,13 @@ class Majordome(Task):
805 795 if type == "WEATHER":
806 796 self.dom.open()
807 797 #scheduler.tasks.scheduling.delay((False, False))
808   - if settings.USE_CELERY:
809   - print("MJ: call schedule WITH CELERY")
810   - scheduler.tasks.scheduling.delay((False, False))
811   - else:
812   - print("MJ: call schedule WITHOUT CELERY")
813   - scheduler.tasks.scheduling().run((False, False))
  798 + print("MJ: call schedule WITHOUT CELERY")
  799 + scheduler.tasks.scheduling().run((False, False))
814 800  
815 801 elif type == "INSIDE":
816 802 #scheduler.tasks.scheduling.delay((False, False))
817   - if settings.USE_CELERY:
818   - print("MJ: call schedule WITH CELERY")
819   - scheduler.tasks.scheduling.delay((False, False))
820   - else:
821   - print("MJ: call schedule WITHOUT CELERY")
822   - scheduler.tasks.scheduling().run((False, False))
  803 + print("MJ: call schedule WITHOUT CELERY")
  804 + scheduler.tasks.scheduling().run((False, False))
823 805  
824 806 '''
825 807 Handle a new alarm (called by isInsideOk or isWeatherOk)
... ... @@ -915,14 +897,9 @@ class Majordome(Task):
915 897 if (self.isValidStatus(self.status_nir)):
916 898 for plan in sequence.albums.get(detector__name="Cagire").plans.all():
917 899 #res = observation_manager.tasks.execute_plan_nir.apply_async((plan.id, float(self.getCountdown(shs))))
918   - if settings.USE_CELERY:
919   - print("MJ: call observation_manager WITH CELERY")
920   - res = observation_manager.tasks.execute_plan_nir.apply_async(
921   - (plan.id, float(self.getCountdown(shs))))
922   - else:
923   - print("MJ: call observation_manager WITHOUT CELERY")
924   - res = observation_manager.tasks.execute_plan_nir().run(
925   - (plan.id, float(self.getCountdown(shs))))
  900 + print("MJ: call observation_manager WITHOUT CELERY")
  901 + res = observation_manager.tasks.execute_plan_nir().run(
  902 + (plan.id, float(self.getCountdown(shs))))
926 903  
927 904 # JB TODO : is it still usefull ?
928 905 # TaskId.objects.create(task_id=res.id, task="execute_plan")
... ... @@ -938,12 +915,8 @@ class Majordome(Task):
938 915 if (self.isValidStatus(self.status_vis)):
939 916 for plan in sequence.albums.get(detector__name="Visible camera").plans.all():
940 917 #res = observation_manager.tasks.execute_plan_vis.apply_async((plan.id, float(self.getCountdown(shs))))
941   - if settings.USE_CELERY:
942   - print("MJ: call observation_manager WITH CELERY")
943   - res = observation_manager.tasks.execute_plan_vis.apply_async((plan.id, float(self.getCountdown(shs))))
944   - else:
945   - print("MJ: call observation_manager WITHOUT CELERY")
946   - res = observation_manager.tasks.execute_plan_vis().run((plan.id, float(self.getCountdown(shs))))
  918 + print("MJ: call observation_manager WITHOUT CELERY")
  919 + res = observation_manager.tasks.execute_plan_vis().run((plan.id, float(self.getCountdown(shs))))
947 920  
948 921 plans_results.append(res)
949 922 else:
... ... @@ -1027,12 +1000,8 @@ class Majordome(Task):
1027 1000 self.logDB("System in pause for " + str(duration))
1028 1001 time.sleep(duration)
1029 1002 #scheduler.tasks.scheduling.apply_async(first_schedule=False, alert=False)
1030   - if settings.USE_CELERY:
1031   - print("MJ: call schedule WITH CELERY")
1032   - scheduler.tasks.scheduling.apply_async(first_schedule=False, alert=False)
1033   - else:
1034   - print("MJ: call schedule WITHOUT CELERY")
1035   - scheduler.tasks.scheduling().run(first_schedule=False, alert=False)
  1003 + print("MJ: call schedule WITHOUT CELERY")
  1004 + scheduler.tasks.scheduling().run(first_schedule=False, alert=False)
1036 1005  
1037 1006 self.setTime()
1038 1007 print("system has been paused. Cause : " + cause)
... ...
src/misc/static/js/plotter.js
  Diff comments   View file @f48ec0d
... ... @@ -28,13 +28,23 @@ function PlotterClas(hkname, plotdiv, HKRange) {
28 28  
29 29 this.layout = {
30 30 title: 'Title of the Graph',
31   - xaxis: {
32   - title: 'Time'
  31 + //xaxis: {
  32 + // title: 'Time'
  33 + //},
  34 + //yaxis: {
  35 + // title: 'Value',
  36 + // type: 'log'
  37 + //},
  38 + width: 400,
  39 + height: 200,
  40 + margin: {
  41 + l: 20,
  42 + r: 10,
  43 + b: 50,
  44 + t: 50,
  45 + pad: 4
33 46 },
34   - yaxis: {
35   - title: 'Value',
36   - type: 'log'
37   - }
  47 + paper_bgcolor: '#f4f4f8'
38 48 };
39 49 this.layout.title = this.hkname;
40 50 this.HKQueue.x = [];
... ...
src/misc/static/js/weather.js
  Diff comments   View file @f48ec0d
1 1  
  2 +/*
  3 +[data_[0].fields :
  4 +cloud: null
  5 +global_status: "OK"
  6 +humidity: 0
  7 +pressure: null
  8 +rain: 0
  9 +temperature: 7
  10 +updated: "2019-01-09T09:44:04.702"
  11 +wind: 1.44
  12 +wind_dir: "193.00"
  13 +*/
  14 +
  15 +
2 16 var sync_plots = {
3   - humidity_plot : new PlotterClas("wind", "humidity_plot", [0,30]),
  17 + humidity_plot : new PlotterClas("humidity", "humidity_plot", [0,80]),
  18 + pressure_plot : new PlotterClas("pressure", "pressure_plot", []),
  19 + rain_plot : new PlotterClas("rain", "rain_plot", []),
  20 + temperature_plot : new PlotterClas("temperature", "temperature_plot", []),
  21 + wind_plot : new PlotterClas("wind", "wind_plot", [0,80]),
  22 + wind_dir_plot : new PlotterClas("wind_dir", "wind_dir_plot", []),
4 23 }
5 24  
6 25 var sync_weather = {
... ... @@ -14,7 +33,12 @@ var sync_weather = {
14 33 document.getElementById('global').innerHTML = res;
15 34 //var res = sync_weather.object_to_tree(data_, ' ');
16 35 //document.getElementById('refresh').innerHTML = res;
17   - sync_plots.humidity_plot.refresh_list([data_[0].fields.updated , data_[0].fields.wind]);
  36 + sync_plots.humidity_plot.refresh_list([data_[0].fields.updated , data_[0].fields.humidity]);
  37 + sync_plots.pressure_plot.refresh_list([data_[0].fields.updated , data_[0].fields.pressure]);
  38 + sync_plots.rain_plot.refresh_list([data_[0].fields.updated , data_[0].fields.rain]);
  39 + sync_plots.temperature_plot.refresh_list([data_[0].fields.updated , data_[0].fields.temperature]);
  40 + sync_plots.wind_plot.refresh_list([data_[0].fields.updated , data_[0].fields.wind]);
  41 + sync_plots.wind_dir_plot.refresh_list([data_[0].fields.updated , data_[0].fields.wind_dir]);
18 42 //}
19 43 },
20 44 tuned_state: function(data) {
... ...
src/monitoring/start_celery_worker.py deleted
View file @7200894
... ... @@ -1,46 +0,0 @@
1   -#!/usr/bin/env python3
2   -
3   -import os
4   -import platform
5   -import subprocess
6   -from sys import stderr, exit
7   -
8   -
9   -def ctrl_c_catch(proc):
10   - print("Ctrl-c caught")
11   - print("\nKilling process...")
12   - proc.kill()
13   - print("Exiting...")
14   - exit(0)
15   -
16   -def main():
17   - if platform.system() == 'Darwin':
18   - p = os.system("brew services list | grep \"rabbitmq started\"")
19   - if (p != 0):
20   - print("RabbitMQ is not started", file=stderr)
21   - return False
22   - else: print("RabbitMQ started: OK")
23   - elif platform.system() == "Linux":
24   - output = subprocess.getoutput("ps -A")
25   - if "rabbitmq-server" in output:
26   - print("RabbitMQ started: OK")
27   - else:
28   - print("RabbitMQ is not started", file=stderr)
29   - return False
30   -
31   -
32   - os.chdir('..')
33   -
34   - try:
35   - process = subprocess.Popen("celery -A pyros worker -Q monitoring_q -n pyros@monitoring -c 1", shell=True)
36   - process.wait()
37   - print("Celery worker stopped")
38   - exit(0)
39   - except KeyboardInterrupt:
40   - ctrl_c_catch(process)
41   -
42   -
43   -
44   - return True
45   -if __name__ == '__main__':
46   - main()
src/monitoring/stop_celery_worker.py deleted
View file @7200894
... ... @@ -1,49 +0,0 @@
1   -#!/usr/bin/env python3
2   -
3   -
4   -# OLD BASH SCRIPT
5   -# ---------------
6   -
7   -# (1) Empty only monitoring queue:
8   -#celery -A pyros amqp queue.purge monitoring_q
9   -
10   -# (2) Empty ALL queues
11   -#celery -A pyros purge -f
12   -
13   -# (3) THE HARD WAY : something like that...
14   -#ps aux | grep \"celery worker\" | awk '{print $2}' | xargs kill -9
15   -
16   -
17   -
18   -# NEW PYTHON SCRIPT
19   -# -----------------
20   -
21   -import subprocess
22   -import sys
23   -import platform
24   -
25   -def main():
26   - # if len(sys.argv) > 1:
27   - #
28   - # if sys.argv[1] == "--help":
29   - # print("stop_celery_worker.py <arg>\n\t--all: purge all queue\n\t--hard: kill all celery worker "
30   - # "\n\t--soft or no arg: purge monitoring queue (this is the standard way)\n\t--help:display this help")
31   - # return True
32   - # elif sys.argv[1] == "--all":
33   - # p = subprocess.Popen("celery -A pyros purge -f", shell=True)
34   - # elif sys.argv[1] == "--hard":
35   - if platform.system() == 'Windows':
36   - p = subprocess.Popen("taskkill /f /im celery.exe", shell=True)
37   - else:
38   - p = subprocess.Popen("ps aux | grep \"monitoring_q\" | awk '{print $2}' | xargs kill -9", shell=True)
39   - # else:
40   - # p = subprocess.Popen("celery -A pyros amqp queue.purge monitoring_q", shell=True)
41   - p.wait()
42   - # if (p.returncode != 0):
43   - # print("ERROR !: Worker's launch failed\r\n", file=sys.stderr)
44   - # return False
45   - print("Workers stopped: OK")
46   - return True
47   -
48   -if __name__ == '__main__':
49   - main()
src/monitoring/tasks.py
  Diff comments   View file @f48ec0d
... ... @@ -2,20 +2,13 @@ from __future__ import absolute_import
2 2 from django.conf import settings
3 3 from common.models import *
4 4  
5   -# (EP) OLD task base class
6   -from celery.task import Task
7   -# NEW task base class, but DOES NOT WORK (because celery3 and not 4 ?) !!!
8   -#from celery import Task
9   -
10 5 from src.monitoring.plc_checker import PlcChecker
11 6 from devices.PLC import PLCController
12 7 from utils.JDManipulator import *
13 8 import json
14 9 import utils.Logger as L
15 10 import majordome.tasks
16   -from celery import app, task
17 11 import alert_manager.tasks
18   -from celery import shared_task
19 12 log = L.setupLogger("MonitoringTaskLogger", "Monitoring")
20 13  
21 14 # EP
... ... @@ -29,7 +22,7 @@ DEBUG_FILE = True
29 22 '''
30 23  
31 24  
32   -class Monitoring(Task):
  25 +class Monitoring():
33 26 timers = {}
34 27 functions = {}
35 28  
... ... @@ -120,31 +113,14 @@ class Monitoring(Task):
120 113 #self.monitoring_task = TaskId.objects.get(task="majordome")
121 114 self.majordome_task = TaskId.objects.get(task="majordome")
122 115 except Exception as e:
123   - if settings.USE_CELERY:
124   - try:
125   - # Mind this will raise an Exception if RABBITMQ is not started !
126   - majordome.tasks.Majordome.apply_async()
127   - if settings.DEBUG and DEBUG_FILE:
128   - log.info(str(e))
129   - except Exception as e2:
130   - print("YOU MUST START RABBITMQ before running PyROS !")
131   - exit()
132   - else: print("USE_CELERY is false => do not create Majordome task")
  116 + print("no USE_CELERY => do not create Majordome task")
133 117  
134 118 if self.alert_task is None:
135 119 try:
136 120 self.alert_task = TaskId.objects.get(task="alert_manager")
137 121 except Exception as e:
138 122 # Mind this will raise an Exception if RABBITMQ is not started !
139   - if settings.USE_CELERY:
140   - try:
141   - alert_manager.tasks.AlertListener.apply_async()
142   - if settings.DEBUG and DEBUG_FILE:
143   - log.info(str(e))
144   - except Exception as e2:
145   - print("YOU MUST START RABBITMQ before running PyROS !")
146   - exit()
147   - else: print("USE_CELERY is false => do not create AlertListener task")
  123 + print("no USE_CELERY => do not create AlertListener task")
148 124 # EP non car 0 = false
149 125 #return 0
150 126 return True
... ...
src/observation_manager/tasks.py
  Diff comments   View file @f48ec0d
1 1 from __future__ import absolute_import
2   -from celery.task import Task
3 2 from django.conf import settings
4 3 from common.models import *
5 4 from utils.Logger import setupLogger
... ... @@ -15,7 +14,8 @@ import os
15 14 '''
16 15 Super class for execute_plan_vis / _nir
17 16 '''
18   -class execute_plan(Task):
  17 +#class execute_plan(Task):
  18 +class execute_plan():
19 19 def setTelescope(self):
20 20 self.tel = TelescopeController()
21 21 return 0
... ... @@ -143,7 +143,8 @@ class execute_plan_nir(execute_plan):
143 143 Call a process with a folder and an image number as parameter who will create the
144 144 calibration for an image
145 145 '''
146   -class create_calibrations(Task):
  146 +#class create_calibrations(Task):
  147 +class create_calibrations():
147 148 logger = setupLogger("calibrations", "calibrations")
148 149  
149 150 def log(self, message: str) -> int:
... ... @@ -210,6 +211,7 @@ class create_calibrations(Task):
210 211 self.execute()
211 212 return self.end()
212 213  
213   -class night_calibrations(Task):
  214 +#class night_calibrations(Task):
  215 +class night_calibrations():
214 216 def run(self):
215 217 return 0
216 218 \ No newline at end of file
... ...
src/pyros/__init__.py deleted
View file @7200894
... ... @@ -1,29 +0,0 @@
1   -from __future__ import absolute_import, unicode_literals
2   -
3   -
4   -# This will make sure the app is always imported when
5   -# Django starts so that shared_task will use this app.
6   -from .celery import app as celery_app
7   -
8   -# EP : normalement aussi, toutes ces autres lignes sont inutiles, à tester
9   -from celery.signals import worker_ready
10   -from django.conf import settings
11   -
12   -# EP : normalement, il faudrait a jouter cette ligne, à tester
13   -#__all__ = ['celery_app']
14   -
15   -
16   -# (if CELERY on) Start agents
17   -if settings.USE_CELERY:
18   - @worker_ready.connect
19   - def start_permanent_tasks(signal, sender):
20   - import monitoring.tasks
21   - import alert_manager.tasks
22   - import majordome.tasks
23   -
24   - if sender.hostname == "pyros@monitoring":
25   - monitoring.tasks.Monitoring.delay()
26   - elif sender.hostname == "pyros@majordome":
27   - majordome.tasks.Majordome.delay()
28   - elif sender.hostname == "pyros@alert_listener":
29   - alert_manager.tasks.AlertListener.delay()
src/pyros/celery.py deleted
View file @7200894
... ... @@ -1,52 +0,0 @@
1   -# First we import absolute_imports from the future, so that our celery.py module won’t clash with the library
2   -from __future__ import absolute_import, unicode_literals
3   -
4   -import os
5   -from celery import Celery
6   -
7   -# TODO: normalement, pas necessaire : a virer ?
8   -from django.conf import settings
9   -
10   -
11   -# Set the default DJANGO_SETTINGS_MODULE environment variable for the celery command-line program
12   -# We don’t really need this line, but it saves us from always passing in the settings module to the celery program.
13   -# It must always come before creating the app instances, as is what we do next
14   -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'pyros.settings')
15   -
16   -app = Celery('pyros')
17   -
18   -# Using a string here means the worker doesn't have to serialize
19   -# the configuration object to child processes.
20   -# - namespace='CELERY' means all celery-related configuration keys
21   -# should have a `CELERY_` prefix.
22   -app.config_from_object('django.conf:settings')
23   -# Celery 4:
24   -#app.config_from_object('django.conf:settings', namespace='CELERY')
25   -
26   -
27   -''' Load task modules from all registered Django app configs.
28   -This is a common practice for reusable apps : we define all tasks in a separate tasks.py module
29   -(1 for each django app), and Celery autodiscovers them
30   -'''
31   -# EP : normalement, la 1ere ligne devrait suffire, à tester
32   -#app.autodiscover_tasks()
33   -app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
34   -
35   -''' Celery will automatically discover tasks from all our installed apps, following the tasks.py convention:
36   -- app1/
37   - - tasks.py
38   - - models.py
39   -- app2/
40   - - tasks.py
41   - - models.py
42   -
43   -This way we don’t have to manually add the individual modules to the CELERY_IMPORTS setting
44   -'''
45   -
46   -
47   -# Thisdebug_task example is a task that dumps its own request information.
48   -# This is using the new bind=True task option introduced in Celery 3.1
49   -# to easily refer to the current task instance (with 'self')
50   -@app.task(bind=True)
51   -def debug_task(self):
52   - print("Request: {0!r}".format(self.request))
src/pyros/settings.py
  Diff comments   View file @f48ec0d
... ... @@ -16,19 +16,11 @@ https://docs.djangoproject.com/en/1.9/ref/settings/
16 16 # FOR SIMULATOR (TODO: remove because not used)
17 17 SIMULATOR = False
18 18  
19   -# FOR SIMULATOR (and TESTS)
20   -CELERY_TEST = False
21   -
22 19 # For majordome_test.py:
23 20 # cd src/majordome/
24 21 # ./majordome_test.py
25 22 MAJORDOME_TEST = False
26 23  
27   -# (EP) Remove this the day we succeed to use no more of Celery...
28   -# Set this to False if you want to run pyros without using Celery (and RabbitMQ)
29   -USE_CELERY = False
30   -#USE_CELERY = True
31   -
32 24 # Set MYSQL to False if you want to use SQLITE
33 25 # This line MUST NOT be changed at all except from changing True/False
34 26 # (or install_requirements script will become invalid)
... ... @@ -172,8 +164,8 @@ else:
172 164 }
173 165 }
174 166  
175   -# CELERY_TEST==True ==> 'TEST' RUN MODE, use pyros_test database
176   -if CELERY_TEST:
  167 +# SIMULATOR==True ==> 'TEST (simu)' RUN MODE, use pyros_test database
  168 +if SIMULATOR:
177 169 DATABASES = {
178 170 'default': {
179 171 'OPTIONS': mysql_options,
... ... @@ -236,7 +228,6 @@ USE_L10N = True
236 228 # Necessary for ENV monitoring :
237 229 USE_TZ = False
238 230 #USE_TZ = True
239   -#if USE_CELERY: USE_TZ = True
240 231  
241 232 # To find the media files {{ MEDIA_URL }}
242 233 MEDIA_URL = '/public/static/media/'
... ... @@ -280,64 +271,3 @@ else:
280 271  
281 272 # from django.core.cache import cache
282 273 # cache.clear()
283   -
284   -# CELERY CONFIG
285   -
286   -CELERY_RESULT_BACKEND = 'amqp'
287   -#CELERY_RESULT_BACKEND = 'rpc://'
288   -#CELERY_RESULT_BACKEND='djcelery.backends.database:DatabaseBackend',
289   -
290   -# (EP) TODO: This should also be added (according to https://simpleisbetterthancomplex.com/tutorial/2017/08/20/how-to-use-celery-with-django.html)
291   -#CELERY_BROKER_URL = 'amqp://localhost'
292   -#CELERY_BROKER_URL = 'amqp://guest:guest@localhost:5672/'
293   -#CELERY_BROKER_URL = 'django://'
294   -
295   -CELERY_ACCEPT_CONTENT = ['json']
296   -CELERY_TASK_SERIALIZER = 'json'
297   -CELERY_RESULT_SERIALIZER = 'json'
298   -
299   -# EP TODO: no more necessary, to be removed
300   -CELERY_IMPORTS = (
301   - "alert_manager.tasks",
302   - "analyzer.tasks",
303   - "majordome.tasks",
304   - "monitoring.tasks",
305   - "observation_manager.tasks",
306   - "scheduler.tasks",
307   -)
308   - # "userSimulator.tasks",
309   -
310   -# This config allows only 1 process / queue. We replace it by the -c option at celery workers creation.
311   -# CELERYD_CONCURRENCY = 1
312   -
313   -''' Following config is needed for manual purge '''
314   -CELERY_ACKS_LATE = False
315   -CELERYD_PREFETCH_MULTIPLIER = 1
316   -
317   -CELERY_QUEUES = {
318   - "alert_listener_q": {"exchange": "alert_listener_q", "routing_key": "alert_listener_q"},
319   - "monitoring_q": {"exchange": "monitoring_q", "routing_key": "monitoring_q"},
320   - "majordome_q": {"exchange": "majordome_q", "routing_key": "majordome_q"},
321   - "analysis_q": {"exchange": "analysis_q", "routing_key": "analysis_q"},
322   - "scheduling_q": {"exchange": "scheduling_q", "routing_key": "scheduling_q"},
323   - "create_calibrations_q": {"exchange": "create_calibrations_q", "routing_key": "create_calibrations_q"},
324   - "execute_plan_vis_q": {"exchange": "execute_plan_vis_q", "routing_key": "execute_plan_vis_q"},
325   - "execute_plan_nir_q": {"exchange": "execute_plan_nir_q", "routing_key": "execute_plan_nir_q"},
326   - "night_calibrations_q": {"exchange": "night_calibrations_q", "routing_key": "night_calibrations_q"},
327   -}
328   -
329   -CELERY_ROUTES = {
330   - "alert_manager.tasks.AlertListener": {"queue": "alert_listener_q"},
331   - "majordome.tasks.Majordome": {"queue": "majordome_q"},
332   - "monitoring.tasks.Monitoring": {"queue": "monitoring_q"},
333   - "analyzer.tasks.analysis": {"queue": "analysis_q"},
334   - "observation_manager.tasks.execute_plan_vis": {"queue": "execute_plan_vis_q"},
335   - "observation_manager.tasks.execute_plan_nir": {"queue": "execute_plan_nir_q"},
336   - "observation_manager.tasks.create_calibrations": {"queue": "create_calibrations_q"},
337   - "observation_manager.tasks.night_calibrations": {"queue": "night_calibrations_q"},
338   - "scheduler.tasks.scheduling": {"queue": "scheduling_q"}
339   -}
340   - # "userSimulator.tasks.simulator": {"queue": "simulator_q"},
341   -
342   -''' Removes pickle warning '''
343   -CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml']
... ...
src/scheduler/tasks.py
  Diff comments   View file @f48ec0d
1 1 from __future__ import absolute_import
2   -from celery.task import Task
3 2 from scheduler.Scheduler import Scheduler
4 3 from common.models import *
5 4 from utils.JDManipulator import *
... ... @@ -7,7 +6,8 @@ from utils.Logger import setupLogger
7 6  
8 7 log = setupLogger("TaskSched", "TaskSched")
9 8  
10   -class scheduling(Task):
  9 +#class scheduling(Task):
  10 +class scheduling():
11 11  
12 12 #TODO Remove first parameter (useless) -> maybe both ?
13 13 def run(self, first_schedule=False, alert=False):
... ...