added support for rpm packaging and basic support for deb
This commit is contained in:
root
@@ -0,0 +1,119 @@
|
||||
"""
|
||||
This module contains a lot of cruft to handle instantiating a "Huey" object
|
||||
using Django settings. Unlike more flexible python apps, the huey django
|
||||
integration consists of a single global Huey instance configured via the
|
||||
settings module.
|
||||
"""
|
||||
from functools import wraps
|
||||
import sys
|
||||
|
||||
from django.conf import settings
|
||||
from django.db import connection
|
||||
|
||||
from huey import crontab
|
||||
from huey import Huey
|
||||
from huey.utils import load_class
|
||||
|
||||
|
||||
configuration_message = """
|
||||
Configuring Huey for use with Django
|
||||
====================================
|
||||
|
||||
Huey was designed to be simple to configure in the general case. For that
|
||||
reason, huey will "just work" with no configuration at all provided you have
|
||||
Redis installed and running locally.
|
||||
|
||||
On the other hand, you can configure huey manually using the following
|
||||
setting structure. The following example uses Redis on localhost:
|
||||
|
||||
Simply point to a backend:
|
||||
|
||||
HUEY = {
|
||||
'backend': 'huey.backends.redis_backend',
|
||||
'name': 'unique name',
|
||||
'connection': {'host': 'localhost', 'port': 6379}
|
||||
|
||||
'consumer_options': {'workers': 4},
|
||||
}
|
||||
|
||||
If you would like to configure Huey's logger using Django's integrated logging
|
||||
settings, the logger used by consumer is named "huey.consumer".
|
||||
|
||||
For more granular control, you can assign HUEY programmatically:
|
||||
|
||||
HUEY = Huey(RedisBlockingQueue('my-queue'))
|
||||
"""
|
||||
|
||||
def default_queue_name():
|
||||
try:
|
||||
return settings.DATABASE_NAME
|
||||
except AttributeError:
|
||||
return settings.DATABASES['default']['NAME']
|
||||
except KeyError:
|
||||
return 'huey'
|
||||
|
||||
def config_error(msg):
|
||||
print(configuration_message)
|
||||
print('\n\n')
|
||||
print(msg)
|
||||
sys.exit(1)
|
||||
|
||||
def dynamic_import(obj, key, required=False):
|
||||
try:
|
||||
path = obj[key]
|
||||
except KeyError:
|
||||
if required:
|
||||
config_error('Missing required configuration: "%s"' % key)
|
||||
return None
|
||||
try:
|
||||
return load_class(path + '.Components')
|
||||
except ImportError:
|
||||
config_error('Unable to import %s: "%s"' % (key, path))
|
||||
|
||||
try:
|
||||
HUEY = getattr(settings, 'HUEY', None)
|
||||
except:
|
||||
config_error('Error encountered reading settings.HUEY')
|
||||
|
||||
if HUEY is None:
|
||||
try:
|
||||
from huey import RedisHuey
|
||||
except ImportError:
|
||||
config_error('Error: Huey could not import the redis backend. '
|
||||
'Install `redis-py`.')
|
||||
HUEY = RedisHuey(default_queue_name())
|
||||
|
||||
if not isinstance(HUEY, Huey):
|
||||
Queue, DataStore, Schedule, Events = dynamic_import(HUEY, 'backend')
|
||||
name = HUEY.get('name') or default_queue_name()
|
||||
conn = HUEY.get('connection', {})
|
||||
always_eager = HUEY.get('always_eager', False)
|
||||
HUEY = Huey(
|
||||
Queue(name, **conn),
|
||||
DataStore and DataStore(name, **conn) or None,
|
||||
Schedule and Schedule(name, **conn) or None,
|
||||
Events and Events(name, **conn) or None,
|
||||
always_eager=always_eager)
|
||||
|
||||
task = HUEY.task
|
||||
periodic_task = HUEY.periodic_task
|
||||
|
||||
def close_db(fn):
|
||||
"""Decorator to be used with tasks that may operate on the database."""
|
||||
@wraps(fn)
|
||||
def inner(*args, **kwargs):
|
||||
try:
|
||||
return fn(*args, **kwargs)
|
||||
finally:
|
||||
connection.close()
|
||||
return inner
|
||||
|
||||
def db_task(*args, **kwargs):
|
||||
def decorator(fn):
|
||||
return task(*args, **kwargs)(close_db(fn))
|
||||
return decorator
|
||||
|
||||
def db_periodic_task(*args, **kwargs):
|
||||
def decorator(fn):
|
||||
return periodic_task(*args, **kwargs)(close_db(fn))
|
||||
return decorator
|
||||
@@ -0,0 +1,126 @@
|
||||
import imp
|
||||
import sys
|
||||
from optparse import make_option
|
||||
|
||||
from django.conf import settings
|
||||
from django.core.management.base import BaseCommand
|
||||
try:
|
||||
from importlib import import_module
|
||||
except ImportError:
|
||||
from django.utils.importlib import import_module
|
||||
|
||||
try:
|
||||
from django.apps import apps as django_apps
|
||||
HAS_DJANGO_APPS = True
|
||||
except ImportError:
|
||||
# Django 1.6
|
||||
HAS_DJANGO_APPS = False
|
||||
|
||||
from huey.consumer import Consumer
|
||||
from huey.bin.huey_consumer import get_loglevel
|
||||
from huey.bin.huey_consumer import setup_logger
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
"""
|
||||
Queue consumer. Example usage::
|
||||
|
||||
To start the consumer (note you must export the settings module):
|
||||
|
||||
django-admin.py run_huey
|
||||
"""
|
||||
help = "Run the queue consumer"
|
||||
|
||||
option_list = BaseCommand.option_list + (
|
||||
make_option('--periodic', '-p',
|
||||
dest='periodic',
|
||||
action='store_true',
|
||||
help='Enqueue periodic commands'
|
||||
),
|
||||
make_option('--no-periodic', '-n',
|
||||
dest='periodic',
|
||||
action='store_false',
|
||||
help='Do not enqueue periodic commands'
|
||||
),
|
||||
make_option('--workers', '-w',
|
||||
dest='workers',
|
||||
type='int',
|
||||
help='Number of worker threads'
|
||||
),
|
||||
make_option('--delay', '-d',
|
||||
dest='initial_delay',
|
||||
type='float',
|
||||
help='Delay between polling requests'
|
||||
),
|
||||
make_option('--max_delay', '-m',
|
||||
dest='max_delay',
|
||||
type='float',
|
||||
help='Maximum delay between polling requests'
|
||||
),
|
||||
)
|
||||
|
||||
def autodiscover_appconfigs(self):
|
||||
"""Use Django app registry to pull out potential apps with tasks.py module."""
|
||||
module_name = 'tasks'
|
||||
for config in django_apps.get_app_configs():
|
||||
app_path = config.module.__path__
|
||||
try:
|
||||
fp, path, description = imp.find_module(module_name, app_path)
|
||||
except ImportError:
|
||||
continue
|
||||
else:
|
||||
import_path = '%s.%s' % (config.name, module_name)
|
||||
imp.load_module(import_path, fp, path, description)
|
||||
|
||||
def autodiscover_old(self):
|
||||
# this is to find modules named <commands.py> in a django project's
|
||||
# installed apps directories
|
||||
module_name = 'tasks'
|
||||
|
||||
for app in settings.INSTALLED_APPS:
|
||||
try:
|
||||
import_module(app)
|
||||
app_path = sys.modules[app].__path__
|
||||
except AttributeError:
|
||||
continue
|
||||
try:
|
||||
imp.find_module(module_name, app_path)
|
||||
except ImportError:
|
||||
continue
|
||||
import_module('%s.%s' % (app, module_name))
|
||||
app_path = sys.modules['%s.%s' % (app, module_name)]
|
||||
|
||||
def autodiscover(self):
|
||||
"""Switch between Django 1.7 style and old style app importing."""
|
||||
if HAS_DJANGO_APPS:
|
||||
self.autodiscover_appconfigs()
|
||||
else:
|
||||
self.autodiscover_old()
|
||||
|
||||
def handle(self, *args, **options):
|
||||
from huey.djhuey import HUEY
|
||||
try:
|
||||
consumer_options = settings.HUEY['consumer_options']
|
||||
except:
|
||||
consumer_options = {}
|
||||
|
||||
if options['workers'] is not None:
|
||||
consumer_options['workers'] = options['workers']
|
||||
|
||||
if options['periodic'] is not None:
|
||||
consumer_options['periodic'] = options['periodic']
|
||||
|
||||
if options['initial_delay'] is not None:
|
||||
consumer_options['initial_delay'] = options['initial_delay']
|
||||
|
||||
if options['max_delay'] is not None:
|
||||
consumer_options['max_delay'] = options['max_delay']
|
||||
|
||||
self.autodiscover()
|
||||
|
||||
loglevel = get_loglevel(consumer_options.pop('loglevel', None))
|
||||
logfile = consumer_options.pop('logfile', None)
|
||||
setup_logger(loglevel, logfile)
|
||||
|
||||
consumer = Consumer(HUEY, **consumer_options)
|
||||
consumer.run()
|
||||
Reference in New Issue
Block a user