added support for rpm packaging and basic support for deb

This commit is contained in:
root
2017-03-29 18:01:35 +02:00
parent ce758e8129
commit 783e7e6d0d
33 changed files with 3455 additions and 0 deletions
+10
View File
@@ -0,0 +1,10 @@
from huey.tests.backends import *
from huey.tests.consumer import *
from huey.tests.crontab import *
from huey.tests.queue import *
from huey.tests.utils import *
try:
import peewee
from huey.tests.peewee_tests import *
except ImportError:
pass
+170
View File
@@ -0,0 +1,170 @@
from collections import deque
import datetime
import os
import sys
import tempfile
import unittest
from huey.api import Huey
from huey.backends.dummy import DummyDataStore
from huey.backends.dummy import DummyEventEmitter
from huey.backends.dummy import DummyQueue
from huey.backends.dummy import DummySchedule
from huey.utils import EmptyData
from huey.backends.sqlite_backend import SqliteDataStore
from huey.backends.sqlite_backend import SqliteQueue
from huey.backends.sqlite_backend import SqliteSchedule
try:
from huey.backends.redis_backend import RedisDataStore
from huey.backends.redis_backend import RedisEventEmitter
from huey.backends.redis_backend import RedisQueue
from huey.backends.redis_backend import RedisSchedule
except ImportError:
RedisQueue = RedisDataStore = RedisSchedule = RedisEventEmitter = None
try:
from huey.backends.rabbitmq_backend import RabbitQueue, RabbitEventEmitter
except ImportError:
RabbitQueue = RabbitEventEmitter = None
if sys.version_info[0] == 2:
redis_kwargs = {}
else:
redis_kwargs = {'decode_responses': True}
QUEUES = (DummyQueue, RedisQueue, SqliteQueue, RabbitQueue)
DATA_STORES = (DummyDataStore, RedisDataStore, SqliteDataStore, None)
SCHEDULES = (DummySchedule, RedisSchedule, SqliteSchedule, None)
EVENTS = (DummyEventEmitter, RedisEventEmitter, None, RabbitEventEmitter)
class HueyBackendTestCase(unittest.TestCase):
def setUp(self):
self.sqlite_location = tempfile.mkstemp(prefix='hueytest.')[1]
def tearDown(self):
os.unlink(self.sqlite_location)
def test_queues(self):
result_store = DummyDataStore('dummy')
for q in QUEUES:
if not q:
continue
if issubclass(q, SqliteQueue):
queue = q('test', location=self.sqlite_location)
elif issubclass(q, RedisQueue):
queue = q('test', **redis_kwargs)
else:
queue = q('test')
queue.flush()
queue.write('a')
queue.write('b')
self.assertEqual(len(queue), 2)
self.assertEqual(queue.read(), 'a')
self.assertEqual(queue.read(), 'b')
self.assertEqual(queue.read(), None)
queue.write('c')
queue.write('d')
queue.write('c')
queue.write('x')
queue.write('d')
self.assertEqual(len(queue), 5)
self.assertEqual(queue.remove('c'), 2)
self.assertEqual(len(queue), 3)
self.assertEqual(queue.read(), 'd')
self.assertEqual(queue.read(), 'x')
self.assertEqual(queue.read(), 'd')
queue.flush()
test_huey = Huey(queue, result_store)
@test_huey.task()
def test_queues_add(k, v):
return k + v
res = test_queues_add('k', 'v')
self.assertEqual(len(queue), 1)
task = test_huey.dequeue()
test_huey.execute(task)
self.assertEqual(res.get(), 'kv')
res = test_queues_add('\xce', '\xcf')
task = test_huey.dequeue()
test_huey.execute(task)
self.assertEqual(res.get(), '\xce\xcf')
def test_data_stores(self):
for d in DATA_STORES:
if not d:
continue
if issubclass(d, SqliteDataStore):
data_store = d('test', location=self.sqlite_location)
elif issubclass(d, RedisDataStore):
data_store = d('test', **redis_kwargs)
else:
data_store = d('test')
data_store.put('k1', 'v1')
data_store.put('k2', 'v2')
data_store.put('k3', 'v3')
self.assertEqual(data_store.peek('k2'), 'v2')
self.assertEqual(data_store.get('k2'), 'v2')
self.assertEqual(data_store.peek('k2'), EmptyData)
self.assertEqual(data_store.get('k2'), EmptyData)
self.assertEqual(data_store.peek('k3'), 'v3')
data_store.put('k3', 'v3-2')
self.assertEqual(data_store.peek('k3'), 'v3-2')
def test_schedules(self):
for s in SCHEDULES:
if not s:
continue
if issubclass(s, SqliteSchedule):
schedule = s('test', location=self.sqlite_location)
elif issubclass(s, RedisSchedule):
schedule = s('test', **redis_kwargs)
else:
schedule = s('test')
dt1 = datetime.datetime(2013, 1, 1, 0, 0)
dt2 = datetime.datetime(2013, 1, 2, 0, 0)
dt3 = datetime.datetime(2013, 1, 3, 0, 0)
dt4 = datetime.datetime(2013, 1, 4, 0, 0)
# Add to schedule out-of-order to ensure sorting is performed by
# the schedule.
schedule.add('s2', dt2)
schedule.add('s1', dt1)
schedule.add('s4', dt4)
schedule.add('s3', dt3)
# Ensure that asking for a timestamp previous to any item in the
# schedule returns empty list.
self.assertEqual(
schedule.read(dt1 - datetime.timedelta(days=1)),
[])
# Ensure the upper boundary is inclusive of whatever timestamp
# is passed in.
self.assertEqual(schedule.read(dt3), ['s1', 's2', 's3'])
self.assertEqual(schedule.read(dt3), [])
# Ensure the schedule is flushed and an empty schedule returns an
# empty list.
self.assertEqual(schedule.read(dt4), ['s4'])
self.assertEqual(schedule.read(dt4), [])
def test_events(self):
for e in EVENTS:
if not e:
continue
e = e('test')
messages = ['a', 'b', 'c', 'd']
for message in messages:
e.emit(message)
if hasattr(e, '_events'):
self.assertEqual(e._events, deque(['d', 'c', 'b', 'a']))
+441
View File
@@ -0,0 +1,441 @@
from collections import deque
import datetime
import json
import logging
import threading
import time
import unittest
from huey import crontab
from huey import Huey
from huey.backends.dummy import DummyDataStore
from huey.backends.dummy import DummyEventEmitter
from huey.backends.dummy import DummyQueue
from huey.backends.dummy import DummySchedule
from huey.consumer import Consumer
from huey.consumer import WorkerThread
from huey.registry import registry
# Logger used by the consumer.
logger = logging.getLogger('huey.consumer')
# Store some global state.
state = {}
# Create a queue, result store, schedule and event emitter, then attach them
# to a test-only Huey instance.
test_queue = DummyQueue('test-queue')
test_result_store = DummyDataStore('test-queue')
test_schedule = DummySchedule('test-queue')
test_events = DummyEventEmitter('test-queue')
test_huey = Huey(test_queue, test_result_store, test_schedule, test_events)
# Create some test tasks.
@test_huey.task()
def modify_state(k, v):
state[k] = v
return v
@test_huey.task()
def blow_up():
raise Exception('blowed up')
@test_huey.task(retries=3)
def retry_command(k, always_fail=True):
if k not in state:
if not always_fail:
state[k] = 'fixed'
raise Exception('fappsk')
return state[k]
@test_huey.task(retries=3, retry_delay=10)
def retry_command_slow(k, always_fail=True):
if k not in state:
if not always_fail:
state[k] = 'fixed'
raise Exception('fappsk')
return state[k]
@test_huey.periodic_task(crontab(minute='0'))
def every_hour():
state['p'] = 'y'
# Create a log handler that will track messages generated by the consumer.
class TestLogHandler(logging.Handler):
def __init__(self, *args, **kwargs):
self.messages = []
logging.Handler.__init__(self, *args, **kwargs)
def emit(self, record):
self.messages.append(record.getMessage())
class ConsumerTestCase(unittest.TestCase):
def setUp(self):
global state
state = {}
self.orig_pc = registry._periodic_tasks
registry._periodic_commands = [every_hour.task_class()]
self.orig_sleep = time.sleep
time.sleep = lambda x: None
test_huey.queue.flush()
test_huey.result_store.flush()
test_huey.schedule.flush()
test_events._events = deque()
self.consumer = Consumer(test_huey, workers=2)
self.consumer._create_threads()
self.handler = TestLogHandler()
logger.addHandler(self.handler)
logger.setLevel(logging.INFO)
def tearDown(self):
self.consumer.shutdown()
logger.removeHandler(self.handler)
registry._periodic_tasks = self.orig_pc
time.sleep = self.orig_sleep
def assertStatusTask(self, status_task):
parsed = []
i = 0
while i < len(status_task):
event = json.loads(test_events._events[i])
status, task, extra = status_task[i]
self.assertEqual(event['status'], status)
self.assertEqual(event['id'], task.task_id)
for k, v in extra.items():
self.assertEqual(event[k], v)
i += 1
def spawn(self, func, *args, **kwargs):
t = threading.Thread(target=func, args=args, kwargs=kwargs)
t.start()
return t
def run_worker(self, task, ts=None):
worker_t = WorkerThread(
test_huey,
self.consumer.default_delay,
self.consumer.max_delay,
self.consumer.backoff,
self.consumer.utc,
self.consumer._shutdown)
ts = ts or datetime.datetime.utcnow()
worker_t.handle_task(task, ts)
def test_message_processing(self):
self.consumer.worker_threads[0].start()
self.assertFalse('k' in state)
res = modify_state('k', 'v')
res.get(blocking=True)
self.assertTrue('k' in state)
self.assertEqual(res.get(), 'v')
self.assertEqual(len(test_events._events), 2)
self.assertStatusTask([
('finished', res.task, {}),
('started', res.task, {}),
])
def test_worker(self):
modify_state('k', 'w')
task = test_huey.dequeue()
self.run_worker(task)
self.assertEqual(state, {'k': 'w'})
def test_worker_exception(self):
blow_up()
task = test_huey.dequeue()
self.run_worker(task)
self.assertTrue(
'Unhandled exception in worker thread' in self.handler.messages)
self.assertEqual(len(test_events._events), 2)
self.assertStatusTask([
('error', task, {'error': True}),
('started', task, {}),
])
def test_retries_and_logging(self):
# this will continually fail
retry_command('blampf')
for i in reversed(range(4)):
task = test_huey.dequeue()
self.assertEqual(task.retries, i)
self.run_worker(task)
if i > 0:
self.assertEqual(
self.handler.messages[-1],
'Re-enqueueing task %s, %s tries left' % (
task.task_id, i - 1))
self.assertStatusTask([
('enqueued', task, {}),
('retrying', task, {}),
('error', task,{}),
('started', task, {}),
])
last_idx = -2
else:
self.assertStatusTask([
('error', task,{}),
('started', task, {}),
])
last_idx = -1
self.assertEqual(self.handler.messages[last_idx],
'Unhandled exception in worker thread')
self.assertEqual(test_huey.dequeue(), None)
def test_retries_with_success(self):
# this will fail once, then succeed
retry_command('blampf', False)
self.assertFalse('blampf' in state)
task = test_huey.dequeue()
self.run_worker(task)
self.assertEqual(self.handler.messages, [
'Executing %s' % task,
'Unhandled exception in worker thread',
'Re-enqueueing task %s, 2 tries left' % task.task_id])
task = test_huey.dequeue()
self.assertEqual(task.retries, 2)
self.run_worker(task)
self.assertEqual(state['blampf'], 'fixed')
self.assertEqual(test_huey.dequeue(), None)
self.assertStatusTask([
('finished', task, {}),
('started', task, {}),
('enqueued', task, {'retries': 2}),
('retrying', task, {'retries': 3}),
('error', task, {'error': True}),
('started', task, {}),
])
def test_scheduling(self):
dt = datetime.datetime(2011, 1, 1, 0, 0)
dt2 = datetime.datetime(2037, 1, 1, 0, 0)
ad1 = modify_state.schedule(args=('k', 'v'), eta=dt, convert_utc=False)
ad2 = modify_state.schedule(args=('k2', 'v2'), eta=dt2, convert_utc=False)
# dequeue the past-timestamped task and run it.
worker = self.consumer.worker_threads[0]
worker.check_message()
self.assertTrue('k' in state)
# dequeue the future-timestamped task.
worker.check_message()
# verify the task got stored in the schedule instead of executing
self.assertFalse('k2' in state)
self.assertStatusTask([
('scheduled', ad2.task, {}),
('finished', ad1.task, {}),
('started', ad1.task, {}),
])
# run through an iteration of the scheduler
self.consumer.scheduler_t.loop(dt)
# our command was not enqueued and no events were emitted.
self.assertEqual(len(test_queue._queue), 0)
self.assertEqual(len(test_events._events), 3)
# run through an iteration of the scheduler
self.consumer.scheduler_t.loop(dt2)
# our command was enqueued
self.assertEqual(len(test_queue._queue), 1)
self.assertEqual(len(test_events._events), 4)
self.assertStatusTask([
('enqueued', ad2.task, {}),
])
def test_retry_scheduling(self):
# this will continually fail
retry_command_slow('blampf')
cur_time = datetime.datetime.utcnow()
task = test_huey.dequeue()
self.run_worker(task, ts=cur_time)
self.assertEqual(self.handler.messages, [
'Executing %s' % task,
'Unhandled exception in worker thread',
'Re-enqueueing task %s, 2 tries left' % task.task_id,
])
in_11 = cur_time + datetime.timedelta(seconds=11)
tasks_from_sched = test_huey.read_schedule(in_11)
self.assertEqual(tasks_from_sched, [task])
task = tasks_from_sched[0]
self.assertEqual(task.retries, 2)
exec_time = task.execute_time
self.assertEqual((exec_time - cur_time).seconds, 10)
self.assertStatusTask([
('scheduled', task, {
'retries': 2,
'retry_delay': 10,
'execute_time': time.mktime(exec_time.timetuple())}),
('retrying', task, {
'retries': 3,
'retry_delay': 10,
'execute_time': None}),
('error', task, {}),
('started', task, {}),
])
def test_revoking_normal(self):
# enqueue 2 normal commands
r1 = modify_state('k', 'v')
r2 = modify_state('k2', 'v2')
# revoke the first *before it has been checked*
r1.revoke()
self.assertTrue(test_huey.is_revoked(r1.task))
self.assertFalse(test_huey.is_revoked(r2.task))
# dequeue a *single* message (r1)
task = test_huey.dequeue()
self.run_worker(task)
self.assertEqual(len(test_events._events), 1)
self.assertStatusTask([
('revoked', r1.task, {}),
])
# no changes and the task was not added to the schedule
self.assertFalse('k' in state)
# dequeue a *single* message
task = test_huey.dequeue()
self.run_worker(task)
self.assertTrue('k2' in state)
def test_revoking_schedule(self):
global state
dt = datetime.datetime(2011, 1, 1)
dt2 = datetime.datetime(2037, 1, 1)
r1 = modify_state.schedule(args=('k', 'v'), eta=dt, convert_utc=False)
r2 = modify_state.schedule(args=('k2', 'v2'), eta=dt, convert_utc=False)
r3 = modify_state.schedule(args=('k3', 'v3'), eta=dt2, convert_utc=False)
r4 = modify_state.schedule(args=('k4', 'v4'), eta=dt2, convert_utc=False)
# revoke r1 and r3
r1.revoke()
r3.revoke()
self.assertTrue(test_huey.is_revoked(r1.task))
self.assertFalse(test_huey.is_revoked(r2.task))
self.assertTrue(test_huey.is_revoked(r3.task))
self.assertFalse(test_huey.is_revoked(r4.task))
expected = [
#state, schedule
({}, 0),
({'k2': 'v2'}, 0),
({'k2': 'v2'}, 1),
({'k2': 'v2'}, 2),
]
for i in range(4):
estate, esc = expected[i]
# dequeue a *single* message
task = test_huey.dequeue()
self.run_worker(task)
self.assertEqual(state, estate)
self.assertEqual(len(test_huey.schedule._schedule), esc)
# lets pretend its 2037
future = dt2 + datetime.timedelta(seconds=1)
self.consumer.scheduler_t.loop(future)
self.assertEqual(len(test_huey.schedule._schedule), 0)
# There are two tasks in the queue now (r3 and r4) -- process both.
for i in range(2):
task = test_huey.dequeue()
self.run_worker(task, future)
self.assertEqual(state, {'k2': 'v2', 'k4': 'v4'})
def test_revoking_periodic(self):
global state
def loop_periodic(ts):
self.consumer.periodic_t.loop(ts)
for i in range(len(test_queue._queue)):
task = test_huey.dequeue()
self.run_worker(task, ts)
# revoke the command once
every_hour.revoke(revoke_once=True)
self.assertTrue(every_hour.is_revoked())
# it will be skipped the first go-round
dt = datetime.datetime(2011, 1, 1, 0, 0)
loop_periodic(dt)
# it has not been run
self.assertEqual(state, {})
# the next go-round it will be enqueued
loop_periodic(dt)
# our command was run
self.assertEqual(state, {'p': 'y'})
# reset state
state = {}
# revoke the command
every_hour.revoke()
self.assertTrue(every_hour.is_revoked())
# it will no longer be enqueued
loop_periodic(dt)
loop_periodic(dt)
self.assertEqual(state, {})
# restore
every_hour.restore()
self.assertFalse(every_hour.is_revoked())
# it will now be enqueued
loop_periodic(dt)
self.assertEqual(state, {'p': 'y'})
# reset
state = {}
# revoke for an hour
td = datetime.timedelta(seconds=3600)
every_hour.revoke(revoke_until=dt + td)
loop_periodic(dt)
self.assertEqual(state, {})
# after an hour it is back
loop_periodic(dt + td)
self.assertEqual(state, {'p': 'y'})
# our data store should reflect the delay
task_obj = every_hour.task_class()
self.assertEqual(len(test_huey.result_store._results), 1)
self.assertTrue(task_obj.revoke_id in test_huey.result_store._results)
+91
View File
@@ -0,0 +1,91 @@
import datetime
import unittest
from huey import crontab
class CrontabTestCase(unittest.TestCase):
def test_crontab_month(self):
# validates the following months, 1, 4, 7, 8, 9
valids = [1, 4, 7, 8, 9]
validate_m = crontab(month='1,4,*/6,8-9')
for x in range(1, 13):
res = validate_m(datetime.datetime(2011, x, 1))
self.assertEqual(res, x in valids)
def test_crontab_day(self):
# validates the following days
valids = [1, 4, 7, 8, 9, 13, 19, 25, 31]
validate_d = crontab(day='*/6,1,4,8-9')
for x in range(1, 32):
res = validate_d(datetime.datetime(2011, 1, x))
self.assertEqual(res, x in valids)
def test_crontab_hour(self):
# validates the following hours
valids = [0, 1, 4, 6, 8, 9, 12, 18]
validate_h = crontab(hour='8-9,*/6,1,4')
for x in range(24):
res = validate_h(datetime.datetime(2011, 1, 1, x))
self.assertEqual(res, x in valids)
edge = crontab(hour=0)
self.assertTrue(edge(datetime.datetime(2011, 1, 1, 0, 0)))
self.assertFalse(edge(datetime.datetime(2011, 1, 1, 12, 0)))
def test_crontab_minute(self):
# validates the following minutes
valids = [0, 1, 4, 6, 8, 9, 12, 18, 24, 30, 36, 42, 48, 54]
validate_m = crontab(minute='4,8-9,*/6,1')
for x in range(60):
res = validate_m(datetime.datetime(2011, 1, 1, 1, x))
self.assertEqual(res, x in valids)
def test_crontab_day_of_week(self):
# validates the following days of week
# jan, 1, 2011 is a saturday
valids = [2, 4, 9, 11, 16, 18, 23, 25, 30]
validate_dow = crontab(day_of_week='0,2')
for x in range(1, 32):
res = validate_dow(datetime.datetime(2011, 1, x))
self.assertEqual(res, x in valids)
def test_crontab_all_together(self):
# jan 1, 2011 is a saturday
# may 1, 2011 is a sunday
validate = crontab(
month='1,5',
day='1,4,7',
day_of_week='0,6',
hour='*/4',
minute='1-5,10-15,50'
)
self.assertTrue(validate(datetime.datetime(2011, 5, 1, 4, 11)))
self.assertTrue(validate(datetime.datetime(2011, 5, 7, 20, 50)))
self.assertTrue(validate(datetime.datetime(2011, 1, 1, 0, 1)))
# fails validation on month
self.assertFalse(validate(datetime.datetime(2011, 6, 4, 4, 11)))
# fails validation on day
self.assertFalse(validate(datetime.datetime(2011, 1, 6, 4, 11)))
# fails validation on day_of_week
self.assertFalse(validate(datetime.datetime(2011, 1, 4, 4, 11)))
# fails validation on hour
self.assertFalse(validate(datetime.datetime(2011, 1, 1, 1, 11)))
# fails validation on minute
self.assertFalse(validate(datetime.datetime(2011, 1, 1, 4, 6)))
def test_invalid_crontabs(self):
# check invalid configurations are detected and reported
self.assertRaises(ValueError, crontab, minute='61')
self.assertRaises(ValueError, crontab, minute='0-61')
+62
View File
@@ -0,0 +1,62 @@
from contextlib import contextmanager
import unittest
from huey import Huey
from huey.backends.dummy import DummyDataStore
from huey.backends.dummy import DummyQueue
from huey.backends.dummy import DummySchedule
from huey.peewee_helpers import db_periodic_task
from huey.peewee_helpers import db_task
from peewee import *
queue = DummyQueue('test-queue')
schedule = DummySchedule('test-queue')
data_store = DummyDataStore('test-queue')
huey = Huey(queue, data_store, schedule=schedule)
STATE = []
class MockSqliteDatabase(SqliteDatabase):
def record_call(fn):
def inner(*args, **kwargs):
STATE.append(fn.__name__)
return fn(*args, **kwargs)
return inner
connect = record_call(SqliteDatabase.connect)
_close = record_call(SqliteDatabase._close)
transaction = record_call(SqliteDatabase.transaction)
db = MockSqliteDatabase('test.huey.db')
class Value(Model):
data = CharField()
class Meta:
database = db
@classmethod
def create(cls, *args, **kwargs):
STATE.append('create')
return super(Value, cls).create(*args, **kwargs)
@db_task(huey, db)
def test_db_task(val):
return Value.create(data=val)
class TestPeeweeHelpers(unittest.TestCase):
def setUp(self):
global STATE
STATE = []
queue.flush()
data_store.flush()
schedule.flush()
Value.drop_table(True)
Value.create_table()
def test_helper(self):
test_db_task('foo')
self.assertEqual(STATE, ['connect'])
huey.execute(huey.dequeue())
self.assertEqual(STATE, ['connect', 'transaction', 'create', '_close'])
self.assertEqual(Value.select().count(), 1)
+438
View File
@@ -0,0 +1,438 @@
import datetime
import unittest
from huey import crontab
from huey import exceptions as huey_exceptions
from huey import Huey
from huey.api import QueueTask
from huey.backends.dummy import DummyDataStore
from huey.backends.dummy import DummyQueue
from huey.backends.dummy import DummySchedule
from huey.registry import registry
from huey.utils import EmptyData
from huey.utils import local_to_utc
queue_name = 'test-queue'
queue = DummyQueue(queue_name)
schedule = DummySchedule(queue_name)
huey = Huey(queue, schedule=schedule)
res_queue_name = 'test-queue-2'
res_queue = DummyQueue(res_queue_name)
res_store = DummyDataStore(res_queue_name)
res_huey = Huey(res_queue, res_store, schedule)
res_huey_nones = Huey(res_queue, res_store, store_none=True)
# store some global state
state = {}
last_executed_task_class = []
# create a decorated queue command
@huey.task()
def add(key, value):
state[key] = value
@huey.task(include_task=True)
def self_aware(key, value, task=None):
last_executed_task_class.append(task.__class__.__name__)
# create a periodic queue command
@huey.periodic_task(crontab(minute='0'))
def add_on_the_hour():
state['periodic'] = 'x'
# define a command using the class
class AddTask(QueueTask):
def execute(self):
k, v = self.data
state[k] = v
# create a command that raises an exception
class BampfException(Exception):
pass
@huey.task()
def throw_error():
raise BampfException('bampf')
@res_huey.task()
def add2(a, b):
return a + b
@res_huey.periodic_task(crontab(minute='0'))
def add_on_the_hour2():
state['periodic'] = 'x'
@res_huey.task()
def returns_none():
return None
@res_huey_nones.task()
def returns_none2():
return None
class HueyTestCase(unittest.TestCase):
def setUp(self):
global state
global last_executed_task_class
queue.flush()
res_queue.flush()
schedule.flush()
state = {}
last_executed_task_class = []
def test_registration(self):
self.assertTrue('queuecmd_add' in registry)
self.assertTrue('queuecmd_add_on_the_hour' in registry)
self.assertTrue('AddTask' in registry)
def test_enqueue(self):
# sanity check
self.assertEqual(len(queue), 0)
# initializing the command does not enqueue it
ac = AddTask(('k', 'v'))
self.assertEqual(len(queue), 0)
# ok, enqueue it, then check that it was enqueued
huey.enqueue(ac)
self.assertEqual(len(queue), 1)
# it can be enqueued multiple times
huey.enqueue(ac)
self.assertEqual(len(queue), 2)
# no changes to state
self.assertFalse('k' in state)
def test_enqueue_decorator(self):
# sanity check
self.assertEqual(len(queue), 0)
add('k', 'v')
self.assertEqual(len(queue), 1)
add('k', 'v')
self.assertEqual(len(queue), 2)
# no changes to state
self.assertFalse('k' in state)
def test_schedule(self):
dt = datetime.datetime(2011, 1, 1, 0, 1)
add('k', 'v')
self.assertEqual(len(queue), 1)
task = huey.dequeue()
self.assertEqual(task.execute_time, None)
add.schedule(args=('k2', 'v2'), eta=dt)
self.assertEqual(len(queue), 1)
task = huey.dequeue()
self.assertEqual(task.execute_time, local_to_utc(dt))
add.schedule(args=('k3', 'v3'), eta=dt, convert_utc=False)
self.assertEqual(len(queue), 1)
task = huey.dequeue()
self.assertEqual(task.execute_time, dt)
def test_error_raised(self):
throw_error()
# no error
task = huey.dequeue()
# error
self.assertRaises(BampfException, huey.execute, task)
def test_internal_error(self):
"""
Verify that exceptions are wrapped with the special "huey"
exception classes.
"""
class SpecialException(Exception):
pass
class BrokenQueue(DummyQueue):
def read(self):
raise SpecialException('read error')
def write(self, data):
raise SpecialException('write error')
class BrokenDataStore(DummyDataStore):
def get(self, key):
raise SpecialException('get error')
def put(self, key, value):
raise SpecialException('put error')
class BrokenSchedule(DummySchedule):
def add(self, data, ts):
raise SpecialException('add error')
def read(self, ts):
raise SpecialException('read error')
task = AddTask()
huey = Huey(
BrokenQueue('q'),
BrokenDataStore('q'),
BrokenSchedule('q'))
self.assertRaises(
huey_exceptions.QueueWriteException,
huey.enqueue,
AddTask())
self.assertRaises(
huey_exceptions.QueueReadException,
huey.dequeue)
self.assertRaises(
huey_exceptions.DataStorePutException,
huey.revoke,
task)
self.assertRaises(
huey_exceptions.DataStoreGetException,
huey.restore,
task)
self.assertRaises(
huey_exceptions.ScheduleAddException,
huey.add_schedule,
task)
self.assertRaises(
huey_exceptions.ScheduleReadException,
huey.read_schedule,
1)
def test_dequeueing(self):
res = huey.dequeue() # no error raised if queue is empty
self.assertEqual(res, None)
add('k', 'v')
task = huey.dequeue()
self.assertTrue(isinstance(task, QueueTask))
self.assertEqual(task.get_data(), (('k', 'v'), {}))
def test_execution(self):
self.assertFalse('k' in state)
add('k', 'v')
task = huey.dequeue()
self.assertFalse('k' in state)
huey.execute(task)
self.assertEqual(state['k'], 'v')
add('k', 'X')
self.assertEqual(state['k'], 'v')
huey.execute(huey.dequeue())
self.assertEqual(state['k'], 'X')
self.assertRaises(TypeError, huey.execute, huey.dequeue())
def test_self_awareness(self):
self_aware('k', 'v')
task = huey.dequeue()
huey.execute(task)
self.assertEqual(last_executed_task_class.pop(), "queuecmd_self_aware")
self_aware('k', 'v')
huey.execute(huey.dequeue())
self.assertEqual(last_executed_task_class.pop(), "queuecmd_self_aware")
add('k', 'x')
huey.execute(huey.dequeue())
self.assertEqual(len(last_executed_task_class), 0)
def test_call_local(self):
self.assertEqual(len(queue), 0)
self.assertEqual(state, {})
add.call_local('nugget', 'green')
self.assertEqual(len(queue), 0)
self.assertEqual(state['nugget'], 'green')
def test_revoke(self):
ac = AddTask(('k', 'v'))
ac2 = AddTask(('k2', 'v2'))
ac3 = AddTask(('k3', 'v3'))
res_huey.enqueue(ac)
res_huey.enqueue(ac2)
res_huey.enqueue(ac3)
res_huey.enqueue(ac2)
res_huey.enqueue(ac)
self.assertEqual(len(res_queue), 5)
res_huey.revoke(ac2)
while res_queue:
task = res_huey.dequeue()
if not res_huey.is_revoked(task):
res_huey.execute(task)
self.assertEqual(state, {'k': 'v', 'k3': 'v3'})
def test_revoke_periodic(self):
add_on_the_hour2.revoke()
self.assertTrue(add_on_the_hour2.is_revoked())
# it is still revoked
self.assertTrue(add_on_the_hour2.is_revoked())
add_on_the_hour2.restore()
self.assertFalse(add_on_the_hour2.is_revoked())
add_on_the_hour2.revoke(revoke_once=True)
self.assertTrue(add_on_the_hour2.is_revoked()) # it is revoked once, but we are preserving that state
self.assertTrue(add_on_the_hour2.is_revoked(peek=False)) # is revoked once, but clear state
self.assertFalse(add_on_the_hour2.is_revoked()) # no longer revoked
d = datetime.datetime
add_on_the_hour2.revoke(revoke_until=d(2011, 1, 1, 11, 0))
self.assertTrue(add_on_the_hour2.is_revoked(dt=d(2011, 1, 1, 10, 0)))
self.assertTrue(add_on_the_hour2.is_revoked(dt=d(2011, 1, 1, 10, 59)))
self.assertFalse(add_on_the_hour2.is_revoked(dt=d(2011, 1, 1, 11, 0)))
add_on_the_hour2.restore()
self.assertFalse(add_on_the_hour2.is_revoked())
def test_result_store(self):
res = add2(1, 2)
res2 = add2(4, 5)
res3 = add2(0, 0)
# none have been executed as yet
self.assertEqual(res.get(), None)
self.assertEqual(res2.get(), None)
self.assertEqual(res3.get(), None)
# execute the first task
res_huey.execute(res_huey.dequeue())
self.assertEqual(res.get(), 3)
self.assertEqual(res2.get(), None)
self.assertEqual(res3.get(), None)
# execute the second task
res_huey.execute(res_huey.dequeue())
self.assertEqual(res.get(), 3)
self.assertEqual(res2.get(), 9)
self.assertEqual(res3.get(), None)
# execute the 3rd, which returns a zero value
res_huey.execute(res_huey.dequeue())
self.assertEqual(res.get(), 3)
self.assertEqual(res2.get(), 9)
self.assertEqual(res3.get(), 0)
# check that it returns None when nothing is present
res = returns_none()
self.assertEqual(res.get(), None)
# execute, it will still return None, but underneath it is an EmptyResult
# indicating its actual result was not persisted
res_huey.execute(res_huey.dequeue())
self.assertEqual(res.get(), None)
self.assertEqual(res._result, EmptyData)
# execute again, this time note that we're pointing at the invoker
# that *does* accept None as a store-able result
res = returns_none2()
self.assertEqual(res.get(), None)
# it stores None
res_huey_nones.execute(res_huey_nones.dequeue())
self.assertEqual(res.get(), None)
self.assertEqual(res._result, None)
def test_task_store(self):
dt1 = datetime.datetime(2011, 1, 1, 0, 0)
dt2 = datetime.datetime(2035, 1, 1, 0, 0)
add2.schedule(args=('k', 'v'), eta=dt1, convert_utc=False)
task1 = res_huey.dequeue()
add2.schedule(args=('k2', 'v2'), eta=dt2, convert_utc=False)
task2 = res_huey.dequeue()
add2('k3', 'v3')
task3 = res_huey.dequeue()
# add the command to the schedule
res_huey.add_schedule(task1)
self.assertEqual(len(res_huey.schedule._schedule), 1)
# add a future-dated command
res_huey.add_schedule(task2)
self.assertEqual(len(res_huey.schedule._schedule), 2)
res_huey.add_schedule(task3)
tasks = res_huey.read_schedule(dt1)
self.assertEqual(tasks, [task3, task1])
tasks = res_huey.read_schedule(dt1)
self.assertEqual(tasks, [])
tasks = res_huey.read_schedule(dt2)
self.assertEqual(tasks, [task2])
def test_ready_to_run_method(self):
dt1 = datetime.datetime(2011, 1, 1, 0, 0)
dt2 = datetime.datetime(2035, 1, 1, 0, 0)
add2.schedule(args=('k', 'v'), eta=dt1)
task1 = res_huey.dequeue()
add2.schedule(args=('k2', 'v2'), eta=dt2)
task2 = res_huey.dequeue()
add2('k3', 'v3')
task3 = res_huey.dequeue()
add2.schedule(args=('k4', 'v4'), task_id='test_task_id')
task4 = res_huey.dequeue()
# sanity check what should be run
self.assertTrue(res_huey.ready_to_run(task1))
self.assertFalse(res_huey.ready_to_run(task2))
self.assertTrue(res_huey.ready_to_run(task3))
self.assertTrue(res_huey.ready_to_run(task4))
self.assertEqual('test_task_id', task4.task_id)
def test_task_delay(self):
curr = datetime.datetime.utcnow()
curr50 = curr + datetime.timedelta(seconds=50)
curr70 = curr + datetime.timedelta(seconds=70)
add2.schedule(args=('k', 'v'), delay=60)
task1 = res_huey.dequeue()
add2.schedule(args=('k2', 'v2'), delay=600)
task2 = res_huey.dequeue()
add2('k3', 'v3')
task3 = res_huey.dequeue()
# add the command to the schedule
res_huey.add_schedule(task1)
res_huey.add_schedule(task2)
res_huey.add_schedule(task3)
# sanity check what should be run
self.assertFalse(res_huey.ready_to_run(task1))
self.assertFalse(res_huey.ready_to_run(task2))
self.assertTrue(res_huey.ready_to_run(task3))
self.assertFalse(res_huey.ready_to_run(task1, curr50))
self.assertFalse(res_huey.ready_to_run(task2, curr50))
self.assertTrue(res_huey.ready_to_run(task3, curr50))
self.assertTrue(res_huey.ready_to_run(task1, curr70))
self.assertFalse(res_huey.ready_to_run(task2, curr70))
self.assertTrue(res_huey.ready_to_run(task3, curr70))
+24
View File
@@ -0,0 +1,24 @@
import unittest
from huey.utils import wrap_exception
class MyException(Exception):
pass
class TestWrapException(unittest.TestCase):
def test_wrap_exception(self):
def raise_keyerror():
try:
{}['huey']
except KeyError as exc:
raise wrap_exception(MyException)
self.assertRaises(MyException, raise_keyerror)
try:
raise_keyerror()
except MyException as exc:
self.assertEqual(str(exc), "KeyError: 'huey'")
else:
assert False