diff options
Diffstat (limited to 'bitbake/lib/toaster')
-rw-r--r-- | bitbake/lib/toaster/dbqueue/__init__.py | 0 | ||||
-rw-r--r-- | bitbake/lib/toaster/dbqueue/celery.py | 17 | ||||
-rw-r--r-- | bitbake/lib/toaster/dbqueue/tasks.py | 25 | ||||
-rw-r--r-- | bitbake/lib/toaster/orm/database_writer.py | 19 | ||||
-rw-r--r-- | bitbake/lib/toaster/toastermain/settings.py | 23 |
5 files changed, 79 insertions, 5 deletions
diff --git a/bitbake/lib/toaster/dbqueue/__init__.py b/bitbake/lib/toaster/dbqueue/__init__.py new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/bitbake/lib/toaster/dbqueue/__init__.py diff --git a/bitbake/lib/toaster/dbqueue/celery.py b/bitbake/lib/toaster/dbqueue/celery.py new file mode 100644 index 00000000000..9de3bcb6818 --- /dev/null +++ b/bitbake/lib/toaster/dbqueue/celery.py @@ -0,0 +1,17 @@ +from __future__ import absolute_import + +import os + +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'toastermain.settings') + +from django.conf import settings # noqa + +app = Celery('dbqueue') + +# Using a string here means the worker will not have to +# pickle the object when using Windows. +app.config_from_object('django.conf:settings') +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
\ No newline at end of file diff --git a/bitbake/lib/toaster/dbqueue/tasks.py b/bitbake/lib/toaster/dbqueue/tasks.py new file mode 100644 index 00000000000..fa874e12d60 --- /dev/null +++ b/bitbake/lib/toaster/dbqueue/tasks.py @@ -0,0 +1,25 @@ +from __future__ import absolute_import +from celery import shared_task +from orm.models import Project + +@shared_task +def save_object(obj): + """ + Call save() on the Django Model object obj + + data: a single Django Model instance in a list (it's in a list because we + need this to serialise the model) + """ + print 'DATABASE WRITER: save_object(); class: %s' % obj.__class__.__name__ + method_to_call = getattr(obj, 'save') + return method_to_call() + +@shared_task +def bulk_create(clazz, data): + print 'DATABASE WRITER: bulk_create(); class: %s' % clazz.__name__ + return clazz.objects.bulk_create(data) + +@shared_task +def get_or_create_default_project(): + print 'DATABASE WRITER: get_or_create_default_project()' + return Project.objects.get_or_create_default_project()
\ No newline at end of file diff --git a/bitbake/lib/toaster/orm/database_writer.py b/bitbake/lib/toaster/orm/database_writer.py index c35c02abc17..3ccda6b39a7 100644 --- a/bitbake/lib/toaster/orm/database_writer.py +++ b/bitbake/lib/toaster/orm/database_writer.py @@ -20,7 +20,10 @@ import collections import logging logger = logging.getLogger("ToasterLogger") +from django.core import serializers + from orm.models import Project +from dbqueue.tasks import save_object, bulk_create, get_or_create_default_project class DatabaseWriter(object): """ @@ -45,25 +48,31 @@ class DatabaseWriter(object): def get_or_create_default_project(self): """ Get or create the default Project model instance - """ logger.info('DATABASE WRITER: get_or_create_default_project()') self.record_method_call('get_or_create_default_project') return Project.objects.get_or_create_default_project() + """ + self.record_method_call('get_or_create_default_project') + return get_or_create_default_project() def save_object(self, obj): """ Call save() on the Django Model object obj - """ logger.info('DATABASE WRITER: save_object(); class: %s' % obj.__class__.__name__) self.record_method_call('save_object') method_to_call = getattr(obj, 'save') return method_to_call() + """ + self.record_method_call('save_object') + return save_object(obj) - def bulk_create(self, clazz, data): + def bulk_create(self, clazz, objs): """ Call bulk_create() on the Django Model class clazz, passing data (list of model instances) - """ logger.info('DATABASE WRITER: bulk_create(); class: %s' % clazz.__name__) self.record_method_call('bulk_create', data) - return clazz.objects.bulk_create(data)
\ No newline at end of file + return clazz.objects.bulk_create(data) + """ + self.record_method_call('bulk_create', objs) + return bulk_create.delay(clazz, objs)
\ No newline at end of file diff --git a/bitbake/lib/toaster/toastermain/settings.py b/bitbake/lib/toaster/toastermain/settings.py index 74ab60462bb..f2b56c45fa8 100644 --- a/bitbake/lib/toaster/toastermain/settings.py +++ b/bitbake/lib/toaster/toastermain/settings.py @@ -38,6 +38,27 @@ ADMINS = ( MANAGERS = ADMINS +# Celery settings +import djcelery +djcelery.setup_loader() + +# adapted from http://redislite.readthedocs.io/en/latest/topic/using_with_existing_modules.html +# monkey patch Redis to use redislite +from redislite import Redis +import redislite.patch +redislite.patch.patch_redis() + +REDIS_DB_PATH = 'toaster.dbqueue.redis' +redislite.patch.patch_redis_Redis(REDIS_DB_PATH) +rdb = Redis(REDIS_DB_PATH) +REDIS_SOCKET_PATH = 'redis+socket://%s' % rdb.socket_file + +BROKER_URL = REDIS_SOCKET_PATH +CELERY_RESULT_BACKEND = REDIS_SOCKET_PATH +CELERY_ACCEPT_CONTENT = ['json'] +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' + DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'. @@ -272,6 +293,8 @@ INSTALLED_APPS = ( 'django.contrib.humanize', 'bldcollector', 'toastermain', + 'djcelery', + 'dbqueue', ) |