PYROS-IRAP / pyros

Commit fd782a47dcc5dc1efc1c716fc0793f2ca356ded8

Authored by Etienne Pallier
1 parent 7f5ddc07
Exists in dev

refactorisations et commentaires (utiles)

Showing 3 changed files with 85 additions and 67 deletions   Show diff stats
Inline Side-by-side
pyros.py
  Diff comments   View file @fd782a4
... ... @@ -469,27 +469,41 @@ class Pyros(AManager):
469 469 self.changeDirectory("..")
470 470 return 0
471 471  
472   - def celery_on(self):
  472 + def celery_on(self, TOTAL=True):
473 473 self.changeDirectory("src")
474   - self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q alert_listener_q -n pyros@alert_listener -c 1")
475   - self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q monitoring_q -n pyros@monitoring -c 1")
  474 +
  475 + if TOTAL: self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q alert_listener_q -n pyros@alert_listener -c 1")
  476 + if TOTAL: self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q monitoring_q -n pyros@monitoring -c 1")
  477 +
  478 + #self.singleWorker("majordome")
476 479 self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q majordome_q -n pyros@majordome -c 1")
477 480  
  481 + #self.singleWorker("scheduling")
478 482 self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q scheduling_q --purge -n pyros@scheduling -c 1")
  483 +
  484 + #self.singleWorker("execute_plan_vis")
479 485 self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q execute_plan_vis_q --purge -n pyros@execute_plan_vis -c 1")
480   - self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q night_calibrations_q --purge -n pyros@night_calibrations -c 1")
  486 +
  487 + if TOTAL: self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q night_calibrations_q --purge -n pyros@night_calibrations -c 1")
  488 +
  489 + #self.singleWorker("execute_plan_nir")
481 490 self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q execute_plan_nir_q --purge -n pyros@execute_plan_nir -c 1")
  491 +
  492 + #self.singleWorker("create_calibrations")
482 493 self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q create_calibrations_q --purge -n pyros@create_calibrations -c 1")
  494 +
  495 + #self.singleWorker("analysis")
483 496 self.execProcessFromVenvAsync(self.venv_cel + " worker -A pyros -Q analysis_q --purge -n pyros@analysis -c 1")
  497 +
484 498 self.changeDirectory("..")
485 499 return 0
486 500  
487   - def start(self):
488   - self.stop()
  501 + def start_workers(self):
  502 + self.stop_workers()
489 503 self.celery_on()
490 504 return 0
491 505  
492   - def stop(self):
  506 + def stop_workers(self):
493 507 if (self.system == "Windows"):
494 508 self.execProcessAsync("taskkill /f /im celery.exe")
495 509 self.execProcessAsync("taskkill /f /im python.exe")
... ... @@ -503,52 +517,8 @@ class Pyros(AManager):
503 517 self.loaddata()
504 518 return 0
505 519  
506   - def simulator_development(self):
507   - self.changeDirectory("src")
508   - self.replacePatternInFile("CELERY_TEST = False", "CELERY_TEST = True", "pyros/settings.py")
509   - self.replacePatternInFile("SIMULATOR = False", "SIMULATOR = True", "pyros/settings.py")
510   - if self.system == "Windows" : self.execProcess("del /f testdb.sqlite3")
511   - else : self.execProcess("rm -f testdb.sqlite3")
512   - self.changeDirectory("..")
513   - self.migrate()
514   - self.loaddata()
515   - self.server()
516   - self.sleep(2)
517   - self.printFullTerm(Colors.WARNING, "SUMMARY")
518   - self.printColor(Colors.GREEN, "The simulator has been successfully initialised")
519   - self.printColor(Colors.GREEN, "The simulator run on a temp database : src/testdb.sqlite3")
520   - self.printColor(Colors.GREEN, "The simulation will be ended by the task 'simulator herself'")
521   - self.printColor(Colors.GREEN, "If you want to shutdown the simulation, please run :")
522   - self.printColor(Colors.GREEN, "CTRL-C or python pyros.py kill_simulation")
523   - self.printColor(Colors.GREEN, "If the simulation isn't correctly killed, please switch the variable")
524   - self.printColor(Colors.GREEN, "CELERY_TEST in src/pyros/settings.py to false")
525   - self.printFullTerm(Colors.WARNING, "SUMMARY")
526   - self.changeDirectory("simulators/config")
527   - self.printColor(Colors.BOLD, "Existing simulations : ", eol='')
528   - sys.stdout.flush()
529   - if (self.system == "Windows") : self.execProcessSilent("dir /B conf*.json")
530   - else: self.execProcessSilent("ls conf*.json")
531   - self.changeDirectory("..")
532   - conf = self.askQuestion("Which simulation do you want to use", default="conf.json")
533   - self.changeDirectory("..")
534   - self.singleWorker("scheduling")
535   - self.singleWorker("majordome")
536   - self.singleWorker("execute_plan_vis")
537   - self.singleWorker("execute_plan_nir")
538   - self.singleWorker("create_calibrations")
539   - self.singleWorker("analysis")
540   - self.sleep(3)
541   - procs = []
542   - self.changeDirectory("simulators")
543   - self.changeDirectory("user")
544   - procs.append(self.execProcessFromVenvAsync(self.venv_bin + " userSimulator.py " + conf))
545   - self.changeDirectory("..")
546   - for p in procs:
547   - p.wait()
548   - self.changeDirectory("..")
549   - self.kill_simulation()
550   - return 0
551 520  
  521 + # Reset the database content
552 522 def reset_database_sim(self):
553 523 self.changeDirectory("src")
554 524 self.replacePatternInFile("CELERY_TEST = False", "CELERY_TEST = True", "pyros/settings.py")
... ... @@ -556,16 +526,22 @@ class Pyros(AManager):
556 526 self.replacePatternInFile("CELERY_TEST = False", "CELERY_TEST = True", "pyros/settings.py")
557 527 self.changeDirectory("..")
558 528  
559   - def simulator(self):
  529 + # Simulation for the scheduler ONLY (and only some celery workers, not all of them)
  530 + def simulator_development(self):
  531 + self.simulator(False)
  532 +
  533 + # Simulation (by default, with ALL simulators and ALL celery workers)
  534 + def simulator(self, TOTAL=True):
560 535 self.changeDirectory("src")
561 536 self.replacePatternInFile("CELERY_TEST = False", "CELERY_TEST = True", "pyros/settings.py")
562 537 self.replacePatternInFile("SIMULATOR = False", "SIMULATOR = True", "pyros/settings.py")
563 538 if self.system == "Windows" : self.execProcess("del /f testdb.sqlite3")
564 539 else : self.execProcess("rm -f testdb.sqlite3")
565 540 self.changeDirectory("..")
566   - self.reset_database_sim()
  541 + if TOTAL: self.reset_database_sim()
567 542 self.migrate()
568 543 self.loaddata()
  544 + # Launch Django web server
569 545 self.server()
570 546 self.sleep(2)
571 547 self.printFullTerm(Colors.WARNING, "SUMMARY")
... ... @@ -575,7 +551,7 @@ class Pyros(AManager):
575 551 self.printColor(Colors.GREEN, "If you want to shutdown the simulation, please run :")
576 552 self.printColor(Colors.GREEN, "CTRL-C or pyros.py kill_simulation")
577 553 self.printColor(Colors.GREEN, "If the simulation isn't correctly killed, please switch the variable")
578   - self.printColor(Colors.GREEN, "CELERY_TEST in src/pyros/settings.py to false")
  554 + self.printColor(Colors.GREEN, "CELERY_TEST in src/pyros/settings.py to False")
579 555 self.printFullTerm(Colors.WARNING, "SUMMARY")
580 556 self.changeDirectory("simulators/config")
581 557 self.printColor(Colors.BOLD, "Existing simulations : ", eol='')
... ... @@ -585,10 +561,41 @@ class Pyros(AManager):
585 561 self.changeDirectory("..")
586 562 conf = self.askQuestion("Which simulation do you want to use", default="conf.json")
587 563 self.changeDirectory("..")
588   - self.celery_on()
  564 + self.celery_on(TOTAL)
589 565 self.sleep(3)
590   - self.sims_launch(conf)
  566 +
  567 + # (DEFAULT NORMAL) TOTAL RUN
  568 + if TOTAL:
  569 + #self.celery_on()
  570 + #self.sleep(3)
  571 + # Launch ALL simulators and WAIT until finished:
  572 + self.sims_launch(conf)
  573 +
  574 + # (DEV) PARTIAL RUN
  575 + else:
  576 + '''
  577 + self.singleWorker("scheduling")
  578 + self.singleWorker("majordome")
  579 + self.singleWorker("execute_plan_vis")
  580 + self.singleWorker("execute_plan_nir")
  581 + self.singleWorker("create_calibrations")
  582 + self.singleWorker("analysis")
  583 + self.sleep(3)
  584 + '''
  585 + procs = []
  586 + self.changeDirectory("simulators")
  587 + self.changeDirectory("user")
  588 + # Launch only 1 simulator : UserSimulator
  589 + procs.append(self.execProcessFromVenvAsync(self.venv_bin + " userSimulator.py " + conf))
  590 + self.changeDirectory("..")
  591 + for p in procs:
  592 + p.wait()
  593 + self.changeDirectory("..")
  594 +
  595 + # When simulators are finished:
  596 + self.kill_simulation()
591 597 return 0
  598 +
592 599  
593 600 def kill_server(self):
594 601 self.execProcessAsync("fuser -k 8000/tcp")
... ... @@ -604,7 +611,10 @@ class Pyros(AManager):
604 611 self.changeDirectory("..")
605 612 return 0
606 613 else: self.execProcessAsync("rm -f testdb.sqlite3")
607   - self.execProcessAsync("fuser -k 8000/tcp")
  614 + # Kill web server (processes using TCP port 8000) :
  615 + #self.execProcessAsync("fuser -k 8000/tcp")
  616 + self.kill_server()
  617 + # Kill all simulators :
608 618 self.execProcessAsync("ps aux | grep \" domeSimulator.py\" | awk '{ print $2 }' | xargs kill")
609 619 self.execProcessAsync("ps aux | grep \" userSimulator.py\" | awk '{ print $2 }' | xargs kill")
610 620 self.execProcessAsync("ps aux | grep \" alertSimulator.py\" | awk '{ print $2 }' | xargs kill")
... ... @@ -613,10 +623,11 @@ class Pyros(AManager):
613 623 self.execProcessAsync("ps aux | grep \" cameraNIRSimulator.py\" | awk '{ print $2 }' | xargs kill")
614 624 self.execProcessAsync("ps aux | grep \" cameraVISSimulator.py\" | awk '{ print $2 }' | xargs kill")
615 625 self.changeDirectory("..")
616   - self.stop()
  626 + self.stop_workers()
617 627 self.printFullTerm(Colors.GREEN, "simulation ended")
618 628 return 0
619 629  
  630 + # Simulators only
620 631 def sims_launch(self, conf=""):
621 632 procs = []
622 633 self.changeDirectory("simulators/config")
... ... @@ -654,7 +665,7 @@ class Pyros(AManager):
654 665 self.changeDirectory("..")
655 666 for p in procs:
656 667 p.wait()
657   - self.kill_simulation()
  668 + #self.kill_simulation()
658 669 return 0
659 670  
660 671 def singleWorker(self, worker):
... ... @@ -700,8 +711,8 @@ class Pyros(AManager):
700 711 "init_database": self.init_database,
701 712 "kill_server": self.kill_server,
702 713 "loaddata": self.loaddata,
703   - "start": self.start,
704   - "stop": self.stop,
  714 + "start_workers": self.start_workers,
  715 + "stop_workers": self.stop_workers,
705 716 "simulator": self.simulator,
706 717 "kill_simulation": self.kill_simulation,
707 718 "sims_launch": self.sims_launch,
... ... @@ -728,8 +739,8 @@ class Pyros(AManager):
728 739 "unittest": "Runs the tests that don't need celery",
729 740 "test_all": "Run all the existing tests (this command needs to be updated when tests are added in the project",
730 741 "celery_on": "Starts celery workers",
731   - "start": "Stop the celery workers then the web server",
732   - "stop": "stops the celery workers",
  742 + "start_workers": "Stop the celery workers then the web server",
  743 + "stop_workers": "stops the celery workers",
733 744 "simulator": "Launch a simulation",
734 745 "simulator_development": "Simulation for the scheduler only",
735 746 "kill_simulation": "kill the simulators / celery workers / web server",
... ...
src/monitoring/tasks.py
  Diff comments   View file @fd782a4
... ... @@ -21,6 +21,8 @@ log = L.setupLogger("MonitoringTaskLogger", "Monitoring")
21 21 #DEBUG_FILE = False
22 22 DEBUG_FILE = True
23 23  
  24 +# Set this to False if you want to test this Environment Monitoring agent ALONE, without using Celery (and RabbitMQ)
  25 +# Set this to True if you want to test this Environment Monitoring agent with Celery (and RabbitMQ)
24 26 USE_CELERY = False
25 27  
26 28  
... ...
src/pyros/settings.py
  Diff comments   View file @fd782a4
... ... @@ -116,12 +116,15 @@ FIXTURE_DIRS = (
116 116  
117 117 LOGIN_URL = "/"
118 118  
119   -# FOR SIMULATOR -> do not touch this variable
  119 +''' DO NOT TOUCH THESE VARIABLES
  120 +"$ pyros.py simulator_development" will automatically set them to "True"
  121 +'''
  122 +# FOR SIMULATOR
120 123 SIMULATOR = False
121   -
122   -# FOR TESTS and SIMULATOR -> do not touch this variable
  124 +# FOR TESTS and SIMULATOR
123 125 CELERY_TEST = False
124 126  
  127 +# CELERY_TEST = False ==> DEFAULT (NORMAL) RUN MODE, use pyros (normal) database
125 128 if not CELERY_TEST:
126 129 if not MYSQL:
127 130 DATABASES = {
... ... @@ -139,6 +142,8 @@ if not CELERY_TEST:
139 142 'PASSWORD': 'DjangoPyros',
140 143 }
141 144 }
  145 +
  146 +# CELERY_TEST = False ==> 'TEST' RUN MODE, use pyros_test database
142 147 else:
143 148 DATABASES = {
144 149 'default': {
... ...