Added some useful comments, mainly about Celery, and very small changes

Etienne Pallier
1 parent 947b3dcf
Showing 6 changed files with 55 additions and 289 deletions Show diff stats
simulators/plc/plcSimulator.py
src/monitoring/tasks.py
src/pyros/__init__.py
src/pyros/celery.py
src/pyros/settings.py
src/pyros/settings.py.bak
@@ -140,7 +140,7 @@ class PLCSimulator(Device, StatusManager):
         i = 0
  
         if (self.ended == 0):
-            self.plcPrint("Not entry for telescope found in config file : " + self.config_file)
+            self.plcPrint("No entry for telescope found in config file : " + self.config_file)
             return (0)
         while (True):
             self.updateStatus(i)
 from __future__ import absolute_import
 from django.conf import settings
 from common.models import *
-from celery.task import Task
+
+# (EP) OLD task base class
+#from celery.task import Task
+# NEW task base class
+from celery import Task
+
 from devices.PLC import PLCController
 from utils.JDManipulator import *
 import json
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
  
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
 from .celery import app as celery_app
  
+# EP : normalement, il faudrait ajouter cette ligne, à tester
+#__all__ = ['celery_app']
+
+# EP : normalement aussi, toutes ces autres lignes sont inutiles, à tester
+
 from celery.signals import worker_ready
  
 from django.conf import settings
-from __future__ import absolute_import
+# First we import absolute_imports from the future, so that our celery.py module won’t clash with the library
+from __future__ import absolute_import, unicode_literals
+
 import os
 from celery import Celery
  
-os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'pyros.settings')
-
+# TODO: normalement, pas necessaire : a virer ?
 from django.conf import settings
  
+
+# Set the default DJANGO_SETTINGS_MODULE environment variable for the celery command-line program
+# We don’t really need this line, but it saves us from always passing in the settings module to the celery program. 
+# It must always come before creating the app instances, as is what we do next
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'pyros.settings')
+
 app = Celery('pyros')
  
-app.config_from_object('django.conf:settings')
+# Using a string here means the worker doesn't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+# EP modif:
+#app.config_from_object('django.conf:settings')
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+
+''' Load task modules from all registered Django app configs.
+This is a common practice for reusable apps : we define all tasks in a separate tasks.py module
+(1 for each django app), and Celery autodiscovers them
+'''
+# EP : normalement, la 1ere ligne devrait suffire, à tester
+#app.autodiscover_tasks()
 app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
+''' Celery will automatically discover tasks from all our installed apps, following the tasks.py convention:
+- app1/
+    - tasks.py
+    - models.py
+- app2/
+    - tasks.py
+    - models.py
+    
+This way we don’t have to manually add the individual modules to the CELERY_IMPORTS setting
+'''
+
  
+# Thisdebug_task example is a task that dumps its own request information. 
+# This is using the new bind=True task option introduced in Celery 3.1 
+# to easily refer to the current task instance (with 'self')
 @app.task(bind=True)
 def debug_task(self):
     print("Request: {0!r}".format(self.request))
@@ -236,6 +236,7 @@ CELERY_ACCEPT_CONTENT = [&#39;json&#39;]
 CELERY_TASK_SERIALIZER = 'json'
 CELERY_RESULT_SERIALIZER = 'json'
  
+# EP TODO: no more necessary, to be removed
 CELERY_IMPORTS = (
     "alert_manager.tasks",
     "analyzer.tasks",
@@ -1,282 +0,0 @@
-"""
-Django settings for pyros project.
-
-Generated by 'django-admin startproject' using Django 1.9.4.
-
-For more information on this file, see
-https://docs.djangoproject.com/en/1.9/topics/settings/
-
-For the full list of settings and their values, see
-https://docs.djangoproject.com/en/1.9/ref/settings/
-"""
-
-# Dictionary containing all the versions for the different modules
-# IMPORTANT : I must be updated at every commit !
-
-MODULES_VERSIONS = {
-    "Alert Manager" : "0.2.4",
-    "Analyzer" : "0.1.2",
-    "Dashboard" : "0.1.1",
-    "Majordome" : "0.1.4",
-    "Monitoring" : "0.1.3",
-    "Observation Manager" : "0.1.3",
-    "Routine Manager" : "0.1.2",
-    "Scheduler" : "0.1.2",
-    "User Manager" : "0.1.1",
-    "Device" : "0.1.1"
-}
-
-# Set MYSQL to False if you want to use SQLITE
-# This line MUST NOT be changed at all except from changing True/False
-# (or install_requirements script will become invalid)
-MYSQL = True
-
-import os
-
-# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-
-# Output folder for images
-OUTPUT_FOLDER = os.path.join(BASE_DIR, "../images_folder")
-
-# Quick-start development settings - unsuitable for production
-# See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/
-
-# SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = '0*@w)$rq4x1c2w!c#gn58*$*u$w=s8uw2zpr_c3nj*u%qlxc23'
-
-# SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
-
-ALLOWED_HOSTS = ['localhost']
-
-
-# Application definition
-
-INSTALLED_APPS = [
-    'django.contrib.admin',
-    'django.contrib.auth',
-    'django.contrib.contenttypes',
-    'django.contrib.sessions',
-    'django.contrib.messages',
-    'django.contrib.staticfiles',
-
-    # for using "./manage.py graph_models" with graphviz:
-    # (https://projects.irap.omp.eu/projects/pyros/wiki/Project_Development#django-extensions-and-graphviz-useful-for-generating-an-image-of-all-the-models-and-their-relationships)
-    'django_extensions',
-    'test_without_migrations',
-    'bootstrap3',
-    'dashboard',
-    'scheduler',
-    'common',
-    'alert_manager',
-    'analyzer',
-    'majordome',
-    'monitoring',
-    'observation_manager',
-    'routine_manager',
-    'user_manager',
-    'devices'
-]
-
-MIDDLEWARE_CLASSES = [
-    'django.middleware.security.SecurityMiddleware',
-    'django.contrib.sessions.middleware.SessionMiddleware',
-    'django.middleware.common.CommonMiddleware',
-    'django.middleware.csrf.CsrfViewMiddleware',
-    'django.contrib.auth.middleware.AuthenticationMiddleware',
-    'django.contrib.auth.middleware.SessionAuthenticationMiddleware',
-    'django.contrib.messages.middleware.MessageMiddleware',
-    'django.middleware.clickjacking.XFrameOptionsMiddleware',
-]
-
-ROOT_URLCONF = 'pyros.urls'
-
-TEMPLATES = [
-    {
-        'BACKEND': 'django.template.backends.django.DjangoTemplates',
-        'DIRS': ['misc/templates'],
-        'APP_DIRS': True,
-        'OPTIONS': {
-            'context_processors': [
-                'django.template.context_processors.debug',
-                'django.template.context_processors.request',
-                'django.contrib.auth.context_processors.auth',
-                'django.contrib.messages.context_processors.messages',
-            ],
-        },
-    },
-]
-
-WSGI_APPLICATION = 'pyros.wsgi.application'
-
-FIXTURE_DIRS = (
-    'misc/fixtures/',
-)
-
-LOGIN_URL = "/"
-
-# FOR SIMULATOR -> do not touch this variable
-SIMULATOR = True
-
-# FOR TESTS and SIMULATOR -> do not touch this variable
-CELERY_TEST = False
-
-if not CELERY_TEST:
-    if not MYSQL:
-        DATABASES = {
-            'default': {
-                'ENGINE': 'django.db.backends.sqlite3',
-                'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
-            }
-        }
-    else:
-        DATABASES = {
-            'default': {
-                'ENGINE': 'django.db.backends.mysql',
-                'NAME': 'pyros',
-                'USER': 'pyros',
-                'PASSWORD': 'DjangoPyros',
-            }
-        }
-else:
-    DATABASES = {
-        'default': {
-            'ENGINE': 'django.db.backends.mysql',
-            'NAME': 'pyros_test',
-            'USER': 'pyros',
-            'PASSWORD': 'DjangoPyros'
-        }
-    }
-
-AUTH_USER_MODEL = 'common.PyrosUser'
-
-# Password validation
-# https://docs.djangoproject.com/en/1.9/ref/settings/#auth-password-validators
-
-AUTH_PASSWORD_VALIDATORS = [
-    {
-        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
-    },
-    {
-        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
-    },
-    {
-        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
-    },
-    {
-        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
-    },
-]
-
-
-# Internationalization
-# https://docs.djangoproject.com/en/1.9/topics/i18n/
-
-LANGUAGE_CODE = 'en-us'
-
-TIME_ZONE = 'UTC'
-
-USE_I18N = True
-
-USE_L10N = True
-
-USE_TZ = True
-
-
-# To find the media files {{ MEDIA_URL }}
-MEDIA_URL = '/public/static/media/'
-
-
-# To find the static files in the app/static/ap/... folders
-STATIC_URL = '/public/static/'
-
-# To find the static files in src/static/. Any local directory can be added to this list.
-STATICFILES_DIRS = (
-                    os.path.join(BASE_DIR, "misc", "static"),
-                    )
-
-# Used for deployment (DEBUG = False). Need to run "python manage.py collectstatic" to fill it.
-STATIC_ROOT = os.path.join(os.path.dirname(BASE_DIR), 'public', 'static')
-
-
-# EP added
-if not DEBUG:
-    '''
-    CACHES = {
-        'default': {
-            'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
-            'LOCATION': '127.0.0.1:11211',
-        }
-    }
-    '''
-    CACHES = {
-        'default': {
-            'BACKEND': 'django.core.cache.backends.filebased.FileBasedCache',
-            'LOCATION': '/var/tmp/django_cache',
-        }
-    }
-
-else:
-    CACHES = {
-        'default': {
-            'BACKEND': 'django.core.cache.backends.dummy.DummyCache',
-        }
-    }
-
-# from django.core.cache import cache
-# cache.clear()
-
-# CELERY CONFIG
-
-# CELERY_RESULT_BACKEND = 'rpc://'
-CELERY_RESULT_BACKEND = 'amqp'
-
-CELERY_ACCEPT_CONTENT = ['json']
-CELERY_TASK_SERIALIZER = 'json'
-CELERY_RESULT_SERIALIZER = 'json'
-
-CELERY_IMPORTS = (
-    "alert_manager.tasks",
-    "analyzer.tasks",
-    "majordome.tasks",
-    "monitoring.tasks",
-    "observation_manager.tasks",
-    "scheduler.tasks",
-)
-    # "userSimulator.tasks",
-
-# This config allows only 1 process / queue. We replace it by the -c option at celery workers creation.
-# CELERYD_CONCURRENCY = 1
-
-''' Following config is needed for manual purge '''
-CELERY_ACKS_LATE = False
-CELERYD_PREFETCH_MULTIPLIER = 1
-
-CELERY_QUEUES = {
-    "alert_listener_q": {"exchange": "alert_listener_q", "routing_key": "alert_listener_q"},
-    "monitoring_q": {"exchange": "monitoring_q", "routing_key": "monitoring_q"},
-    "majordome_q": {"exchange": "majordome_q", "routing_key": "majordome_q"},
-    "analysis_q": {"exchange": "analysis_q", "routing_key": "analysis_q"},
-    "scheduling_q": {"exchange": "scheduling_q", "routing_key": "scheduling_q"},
-    "create_calibrations_q": {"exchange": "create_calibrations_q", "routing_key": "create_calibrations_q"},
-    "execute_plan_vis_q": {"exchange": "execute_plan_vis_q", "routing_key": "execute_plan_vis_q"},
-    "execute_plan_nir_q": {"exchange": "execute_plan_nir_q", "routing_key": "execute_plan_nir_q"},
-    "night_calibrations_q": {"exchange": "night_calibrations_q", "routing_key": "night_calibrations_q"},
-}
-
-CELERY_ROUTES = {
-    "alert_manager.tasks.AlertListener": {"queue": "alert_listener_q"},
-    "majordome.tasks.Majordome": {"queue": "majordome_q"},
-    "monitoring.tasks.Monitoring": {"queue": "monitoring_q"},
-    "analyzer.tasks.analysis": {"queue": "analysis_q"},
-    "observation_manager.tasks.execute_plan_vis": {"queue": "execute_plan_vis_q"},
-    "observation_manager.tasks.execute_plan_nir": {"queue": "execute_plan_nir_q"},
-    "observation_manager.tasks.create_calibrations": {"queue": "create_calibrations_q"},
-    "observation_manager.tasks.night_calibrations": {"queue": "night_calibrations_q"},
-    "scheduler.tasks.scheduling": {"queue": "scheduling_q"}
-}
-    # "userSimulator.tasks.simulator": {"queue": "simulator_q"},
-
-''' Removes pickle warning '''
-CELERY_ACCEPT_CONTENT = ['pickle', 'json', 'msgpack', 'yaml']