from __future__ import absolute_import from celery.task import Task from django.core.exceptions import ObjectDoesNotExist from django.db.models import Q import observation_manager import observation_manager.tasks import scheduler import scheduler.tasks as sched_task import utils.Logger as L import monitoring.tasks import alert_manager.tasks from common.models import * from devices.CameraNIR import NIRCameraController from devices.CameraVIS import VISCameraController from devices.Dome import DomeController from devices.PLC import PLCController from devices.Telescope import TelescopeController from majordome.MajordomeDecorators import * from utils.JDManipulator import * DEBUG_FILE = False log = L.setupLogger("MajordomeTaskLogger", "Majordome") ''' Task to handle the execution of the program check the environment status in database check the devices status (telescope / cameras) check if the last schedule made has to be planned launch schedule's sequences ''' class Majordome(Task): loop_speed = 1 julian_div = 86400 executing_sequence = None next_sequence = None status_tel = "" status_nir = "" status_vis = "" status_dom = "" site_status = "OK" weather_status = "OK" timers = {} functions = {} schedule = None available_status = [] current_status = "RUNNING" alarm_list = [] ''' Check if the instrument status is valid ''' def isValidStatus(self, status): # TODO REMOVE COMMENT AND CHANGE WHEN DEFINED # if (status == "" or status == "ERROR" or status == "FAILED" or status == "NOT_SET"): # return (False) return (True) def setContext(self): self.tel = TelescopeController() self.vis_camera = VISCameraController() self.nir_camera = NIRCameraController() self.plc = PLCController() self.dom = DomeController() return (0) def createTask(self): try: # (EP) NO find() method available from Django ORM !!!! #TaskId.objects.find(task="majordome").delete() TaskId.objects.filter(task="majordome").delete() except Exception as e: log.info(str(e)) return 1 TaskId.objects.create(task_id=self.request.id, task="majordome") return 0 def setTasks(self): try: self.monitoring_task = TaskId.objects.get(task="monitoring") self.alert_task = TaskId.objects.get(task="alert_manager") except Exception as e: self.monitoring_task = None self.alert_task = None return 0 ''' Function called by celery task Behavior: Init telescope / cameras set night limits check the software version launch the majordome loop ''' def run(self): self.createTask() self.updateSoftware() self.setContext() self.setTime() self.setTasks() self.loop() ''' Reads the softwares versions in the settings.py, store them in the DB and send them to the IC. ''' def updateSoftware(self): versions = settings.MODULES_VERSIONS for module, version in versions.items(): same_module_versions = Version.objects.filter(module_name=module) if same_module_versions.count() == 0: Version.objects.create(module_name=module, version=version) elif same_module_versions.order_by("-created")[0].version != version: Version.objects.create(module_name=module, version=version) return (0) ''' Loop to wait for the device to be idle with the starting configurations. ''' def waitDevices(self): while self.status_vis == "" and self.status_tel == "" and self.status_nir == "" and self.status_dom == "": self.status_vis = self.vis_camera.getStatus() self.status_nir = self.nir_camera.getStatus() self.status_tel = self.tel.getStatus() self.status_dom = self.dom.getStatus() return (0) ''' Computes the beginning and the end of the following (or current) night set the timers -> maybe put timers in a config file ? ''' def setTime(self): self.night_start = getNightStart() self.night_end = getNightEnd() self.night_start_jd = secondsToJulianDate(getNightStart()) self.night_end_jd = secondsToJulianDate(getNightEnd()) self.timer_night_start = self.night_start - getCurrentTime() self.timer_night_end = self.night_end - getCurrentTime() self.timer_status = 5 self.tasks_timer = 5 self.timer_plc = 2 self.alert_timer = 1 self.timer_schedule = 1 self.timer_sequence = 1 if (self.night_start - 120 > getCurrentTime()): self.timer_night_start = self.night_start - 120 - getCurrentTime() self.timer_night_end = self.night_end - getCurrentTime() if (getCurrentTime() > self.night_start): self.adaptTimers() self.timers = { "status": self.timer_status, "environment": self.timer_plc, "night_start": self.timer_night_start, "night_end": self.timer_night_end, "schedule": self.timer_schedule, "sequence": self.timer_sequence, "tasks": self.tasks_timer } if (settings.DEBUG and DEBUG_FILE): log.info("Majordome started with timers : " + str(self.timers)) # Functions called during the loop self.functions = { "status": self.handleStatusTimer, "environment": self.handleEnvironmentTimer, "night_start": self.handleNightStartTimer, "night_end": self.handleNightEndTimer, "schedule": self.handleScheduleTimer, "sequence": self.handleSequenceTimer, "tasks": self.handleTasks } return (0) ''' Function called by the main loop to handle the task event (check monitoring and alert_manager) ''' def handleTasks(self): if not settings.USE_CELERY: return 0 self.timers["tasks"] = self.tasks_timer if self.monitoring_task is None: try: self.monitoring_task = TaskId.objects.get(task="monitoring") except Exception as e: monitoring.tasks.Monitoring.apply_async() if settings.DEBUG and DEBUG_FILE: log.info(str(e)) if self.alert_task is None: try: self.alert_task = TaskId.objects.get(task="alert_manager") except Exception as e: alert_manager.tasks.AlertListener.apply_async() if settings.DEBUG and DEBUG_FILE: log.info(str(e)) return 0 # TODO adapt timers if the majordome is started during the night or not ? def adaptTimers(self): pass def logDB(self, message: str): Log.objects.create(agent="Majordome", message=message) ''' Infinite loop according to the majordome behavior ''' def loop(self): while (self.current_status != "SHUTDOWN"): print("(MAJOR): start new iteration") minimal_timer = min(self.timers, key=self.timers.get) if (self.timers[minimal_timer] > 0): time.sleep(self.timers[minimal_timer]) self.timers = {key: value - self.timers[minimal_timer] for key, value in self.timers.items()} for timer_name, timer_value in self.timers.items(): if (timer_value <= 0): if timer_name in self.functions: self.logDB("Executing timer " + str(timer_name)) self.functions[timer_name]() else: if (settings.DEBUG and DEBUG_FILE): log.info("Timer : " + str(timer_name) + "is not known by the Majordome") self.logDB("Timer " + str(timer_name) + " unknown") if (settings.DEBUG and DEBUG_FILE): log.info("Timer : " + str(timer_name) + " executed") # EP added because loop is too quick (without CELERY) if not settings.USE_CELERY: time.sleep(2) return (0) ''' Function called by the main loop to handle environment event (PLC info) ''' def handleEnvironmentTimer(self): self.timers["environment"] = self.timer_plc self.handlePLC() return (0) ''' Function called by the main loop to handle the devices status ''' def handleStatusTimer(self): self.timers["status"] = self.timer_status self.status_tel = self.tel.getStatus() self.status_nir = self.nir_camera.getStatus() self.status_vis = self.vis_camera.getStatus() self.status_dom = self.dom.getStatus() self.handleStatus() return 0 ''' Function called by the main loop to check if the executing sequence is finished ''' def handleSequenceTimer(self): self.timers["sequence"] = self.timer_sequence if (self.executing_sequence): self.handleSequence(self.executing_sequence[0], self.executing_sequence[1], self.executing_sequence[2]) return (0) ''' Function called by the main loop to check if there is a new schedule and to execute its sequences ''' def handleScheduleTimer(self): self.timers["schedule"] = self.timer_schedule if (self.isValidStatus(self.status_tel)): if (self.schedule is None): try: self.schedule = Schedule.objects.latest('created') except ObjectDoesNotExist: if (settings.DEBUG and DEBUG_FILE): log.info("No schedule found in database") return (1) else: try: schedule = Schedule.objects.latest('created') except ObjectDoesNotExist: if (settings.DEBUG and DEBUG_FILE): log.info("No schedule found in database") return (1) if (schedule.created != self.schedule.created): self.next_sequence = None self.schedule = schedule self.firstSequenceIsAlert() if (self.schedule): shs_list = self.schedule.shs.filter(Q(status=Sequence.PLANNED) | Q(status=Sequence.PENDING)).order_by('tsp') self.executeSchedule(shs_list) else: self.notifyTelescopeStatus("scheduler") return (0) ''' Function called by handleScheduleTimer, the purpose is to kill the executing sequence if the first sequence in the new planning is from an alert ''' @executingSequenceExist def firstSequenceIsAlert(self): shs_list = self.schedule.shs.filter(Q(status=Sequence.PLANNED) | Q(status=Sequence.PENDING)).order_by('tsp') if shs_list and shs_list[0].sequence.is_alert: if shs_list[0].sequence.created.date() >= (datetime.datetime.now() - datetime.timedelta(seconds=10)).date(): self.killExecutingSequence() return 1 return 0 ''' Function called by the main loop to handle the end of a night ''' def handleNightEndTimer(self): self.timers["night_end"] = getNightEnd() if (self.isValidStatus(self.status_tel)): #observation_manager.tasks.night_calibrations.apply_async() if settings.USE_CELERY: print("MJ: call observation_manager WITH CELERY") observation_manager.tasks.night_calibrations.apply_async() else: print("MJ: call observation_manager WITHOUT CELERY") observation_manager.tasks.night_calibrations().run() else: self.notifyTelescopeStatus("night_end") return (0) ''' Function called by the main loop to handle the beginning of a night ''' def handleNightStartTimer(self): self.timers["night_start"] = getNextNightStart() if self.isOutsideOk(): self.dom.open() self.vis_camera.open_shutter() self.nir_camera.open_shutter() #scheduler.tasks.scheduling.apply_async((False, False)) if settings.USE_CELERY: print("MJ: call schedule WITH CELERY") scheduler.tasks.scheduling.apply_async((False, False)) else: print("MJ: call schedule WITHOUT CELERY") scheduler.tasks.scheduling().run((False, False)) return (0) def notifyTelescopeStatus(self, timer_name): return (self.notifyDeviceStatus("telescope", timer_name, self.status_tel)) def notifyDeviceStatus(self, device_name, timer_name, status): Log.objects.create(agent=device_name, created=datetime.datetime.now(), message="The action : " + str(timer_name) + " has been canceled : Telescope status : " + str(status)) # TODO MAYBE reset some variables and do a scheduling return (0) ''' Execute a schedule ''' def executeSchedule(self, shs_list): for shs in shs_list: # shs_list is sorted by tsp if (self.executableSequence(shs.sequence.status) and self.observable(shs.sequence)): if self.next_sequence is None: self.setNextSequence(shs, shs.sequence) if self.isExecutable() and self.executing_sequence is None: if self.isValidTimer(self.next_sequence[0]): if self.executeSequence(self.next_sequence[0], self.next_sequence[1]) == -1: return -1 if self.next_sequence[0] != shs and self.next_sequence[1] != shs.sequence: self.setNextSequence(shs, shs.sequence) else: self.next_sequence = None else: return 0 else: if (settings.DEBUG and DEBUG_FILE): log.info("Sequence cannot be executed : Not observable") self.logDB("Sequence "+shs.sequence.name+" cannot be executed : Not observable") return 0 def observable(self, sequence): if (sequence.jd2 - sequence.duration - secondsToPreciseJulianDate(getPreciseCurrentTime()) <= 0): return 0 return 1 def executableSequence(self, status): if status == Sequence.PLANNED or status == Sequence.PENDING: return 1 return 0 ''' Kill the executing sequence and set its state to cancelled ''' @executingSequenceExist def killExecutingSequence(self): shs = self.executing_sequence[0] sequence = self.executing_sequence[1] executing_plans = self.executing_sequence[2] shs.status = Sequence.CANCELLED sequence.status = Sequence.CANCELLED shs.save() sequence.save() for rev in executing_plans: if (not rev.failed() and not rev.successful()): rev.revoke(terminate=True) self.executing_sequence = None return 0 def reset(self, type): if type == "WEATHER": self.dom.open() #scheduler.tasks.scheduling.delay((False, False)) if settings.USE_CELERY: print("MJ: call schedule WITH CELERY") scheduler.tasks.scheduling.delay((False, False)) else: print("MJ: call schedule WITHOUT CELERY") scheduler.tasks.scheduling().run((False, False)) elif type == "INSIDE": #scheduler.tasks.scheduling.delay((False, False)) if settings.USE_CELERY: print("MJ: call schedule WITH CELERY") scheduler.tasks.scheduling.delay((False, False)) else: print("MJ: call schedule WITHOUT CELERY") scheduler.tasks.scheduling().run((False, False)) ''' Handle a new alarm (called by isInsideOk or isWeatherOk) ''' @SameAlarmCheck def handleAlarm(self, type, pos=-1): if type == "WEATHER": #TODO send email self.dom.close() self.killExecutingSequence() self.vis_camera.park() self.nir_camera.park() elif type == "INSIDE": #TODO send email self.killExecutingSequence() self.vis_camera.park() self.nir_camera.park() elif type == "ENDED": if len(self.alarm_list) > 0 and pos != -1: ended = self.alarm_list[pos] del self.alarm_list[pos] self.reset(ended) return 0 else: return 1 self.alarm_list.append(type) return 0 ''' for now weather_status and site_status contains something different than OK if the status is critical Later we may have other states to handle ''' def isOutsideOk(self) -> bool: self.handlePLC() if self.weather_status == "OK": if "WEATHER" in self.alarm_list: self.handleAlarm("ENDED", self.alarm_list.index("WEATHER")) return True self.handleAlarm("WEATHER") return False ''' Check the telescope inside status ''' def isInsideOk(self) -> bool: self.handlePLC() if self.site_status == "OK": if "INSIDE" in self.alarm_list: self.handleAlarm("ENDED", self.alarm_list.index("INSIDE")) return True self.handleAlarm("INSIDE") return False def isDevicesOk(self) -> bool: if self.isValidStatus(self.status_tel) and self.isValidStatus(self.status_dom)\ and self.isValidStatus(self.status_vis) and self.isValidStatus(self.status_nir): return True return False def isExecutable(self) -> bool: if self.isValidStatus(self.status_tel) and self.isValidStatus(self.status_dom)\ and self.isOutsideOk() and self.isInsideOk(): return True return False ''' check if the sequence timer is valid for execution, also check if there is a scheduling task running ''' def isValidTimer(self, shs) -> bool: current_countdown = self.getCountdown(shs) if (current_countdown <= JulianSeconds(5)): try: task = TaskId.objects.filter(task="scheduling") if not task: return True return False except: return True return False ''' Launch the observation tasks NIR and VIS associated to a sequence ''' def executeSequence(self, shs, sequence): shs.status = Sequence.EXECUTING sequence.status = Sequence.EXECUTING shs.save() sequence.save() log.info("Executing sequence id = " + str(sequence.pk)) self.logDB("Executing sequence") plans_results = [] if sequence.albums.filter(detector__name="Cagire").exists(): if (self.isValidStatus(self.status_nir)): for plan in sequence.albums.get(detector__name="Cagire").plans.all(): #res = observation_manager.tasks.execute_plan_nir.apply_async((plan.id, float(self.getCountdown(shs)))) if settings.USE_CELERY: print("MJ: call observation_manager WITH CELERY") res = observation_manager.tasks.execute_plan_nir.apply_async( (plan.id, float(self.getCountdown(shs)))) else: print("MJ: call observation_manager WITHOUT CELERY") res = observation_manager.tasks.execute_plan_nir().run( (plan.id, float(self.getCountdown(shs)))) # JB TODO : is it still usefull ? # TaskId.objects.create(task_id=res.id, task="execute_plan") plans_results.append(res) else: self.notifyDeviceStatus("Cagire", "Sequence execution", self.status_nir) sequence.status = Sequence.CANCELLED shs.status = Sequence.CANCELLED shs.save() sequence.save() return (1) if sequence.albums.filter(detector__name="Visible camera").exists(): if (self.isValidStatus(self.status_vis)): for plan in sequence.albums.get(detector__name="Visible camera").plans.all(): #res = observation_manager.tasks.execute_plan_vis.apply_async((plan.id, float(self.getCountdown(shs)))) if settings.USE_CELERY: print("MJ: call observation_manager WITH CELERY") res = observation_manager.tasks.execute_plan_vis.apply_async((plan.id, float(self.getCountdown(shs)))) else: print("MJ: call observation_manager WITHOUT CELERY") res = observation_manager.tasks.execute_plan_vis().run((plan.id, float(self.getCountdown(shs)))) plans_results.append(res) else: self.notifyDeviceStatus("Camera visible", "Sequence execution", self.status_vis) sequence.status = Sequence.CANCELLED shs.status = Sequence.CANCELLED shs.save() sequence.save() return (1) self.executing_sequence = [shs, sequence, plans_results] return (0) ''' Set the next sequence ''' def setNextSequence(self, shs, sequence): sequence.status = Sequence.PENDING shs.status = Sequence.PENDING self.next_sequence = [shs, sequence] sequence.save() shs.save() return (0) ''' Check if the current sequence is finished ''' def handleSequence(self, shs, sequence, executing_plans): count = 0 for res in executing_plans: try: if res.successful() or res.failed(): count += 1 except Exception as e: if DEBUG_FILE and settings.DEBUG: log.info(str(e)) shs.status = Sequence.CANCELLED sequence.status = Sequence.CANCELLED shs.save() sequence.save() for rev in executing_plans: if (not rev.failed() and not rev.successful()): rev.revoke(terminate=True) self.executing_sequence = None return (-1) if count >= len(executing_plans): sequence.status = Sequence.EXECUTED shs.status = Sequence.EXECUTED sequence.save() shs.save() message = "Finished sequence " + str(sequence.pk) + " execution" Log.objects.create(agent="Majordome", message=message) self.executing_sequence = None return (0) ''' Function called to do an action with the devices status ''' def handleStatus(self): telescope = Telescope.objects.first() camera_nir = Detector.objects.get(name="Cagire") camera_vis = Detector.objects.get(name="Visible camera") dome = Dome.objects.get(name="Dome") dome.status = self.status_dom telescope.status = self.status_tel camera_nir.status = self.status_nir camera_vis.status = self.status_vis dome.save() telescope.save() camera_nir.save() camera_vis.save() self.logDB("Instrument status : dome = " + str(self.status_dom) + ", telescope = " + str(self.status_tel) + ", camera_nir = " + str(self.status_nir) + ", camera_vis" + str(self.status_vis)) return (0) ''' Put the majordome in pause ''' def systemPause(self, duration, cause: str): self.logDB("System in pause for " + str(duration)) time.sleep(duration) #scheduler.tasks.scheduling.apply_async(first_schedule=False, alert=False) if settings.USE_CELERY: print("MJ: call schedule WITH CELERY") scheduler.tasks.scheduling.apply_async(first_schedule=False, alert=False) else: print("MJ: call schedule WITHOUT CELERY") scheduler.tasks.scheduling().run(first_schedule=False, alert=False) self.setTime() print("system has been paused. Cause : " + cause) return (0) ''' Function called to do an action with the site status and the wheather status ''' def handlePLC(self): try: site_status = SiteWatch.objects.latest('updated') weather_status = WeatherWatch.objects.latest('updated') self.weather_status = weather_status.global_status self.site_status = site_status.global_status except ObjectDoesNotExist as e: if (settings.DEBUG and DEBUG_FILE): log.info("No site_status or weather_status found in database : " + str(e)) # TODO shutdown everything return 1 return 0 ''' Gets the time before the expected start of the execution. ''' def getCountdown(self, shs): # TODO start sequence as soon as possible (a lot of verifications must be done there) current_time = secondsToPreciseJulianDate(getPreciseCurrentTime()) countdown = shs.tsp - current_time return countdown ''' Change observation conditions ''' def changeObsConditions(self): print("change_obs_conditions") pass