summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLars-Dominik Braun <lars@6xq.net>2018-05-04 15:36:09 +0200
committerLars-Dominik Braun <lars@6xq.net>2018-05-04 16:00:05 +0200
commit1882062b5c5cacea60db922971538f6b4f6a8c27 (patch)
tree0ac05335ac2f126177e4e7f114e32ae056b26b2e
parentb6d198aed3fa17ac28aa62c9679ec2c33dc0a3da (diff)
downloadcrocoite-1882062b5c5cacea60db922971538f6b4f6a8c27.tar.gz
crocoite-1882062b5c5cacea60db922971538f6b4f6a8c27.tar.bz2
crocoite-1882062b5c5cacea60db922971538f6b4f6a8c27.zip
sopel: Use recursive, distributed controller
-rw-r--r--contrib/celerycrocoite.py9
1 files changed, 7 insertions, 2 deletions
diff --git a/contrib/celerycrocoite.py b/contrib/celerycrocoite.py
index b92e5c8..26c35ce 100644
--- a/contrib/celerycrocoite.py
+++ b/contrib/celerycrocoite.py
@@ -128,7 +128,7 @@ def celeryWorker (bot, q):
break
action, trigger, args = item
if action == 'a':
- handle = task.archive.delay (**args)
+ handle = task.controller.delay (**args)
j = jobs[handle.id] = {'handle': handle, 'trigger': trigger, 'args': args}
# pretty-print a few selected args
@@ -136,6 +136,8 @@ def celeryWorker (bot, q):
'idleTimeout': prettyTimeDelta (args['settings']['idleTimeout']),
'timeout': prettyTimeDelta (args['settings']['timeout']),
'maxBodySize': prettyBytes (args['settings']['maxBodySize']),
+ 'recursive': args['recursive'],
+ 'concurrency': args['concurrency'],
}
strargs = ', '.join (map (lambda x: '{}={}'.format (*x), showargs.items ()))
bot.msg (trigger.sender, '{}: {} has been queued as {} with {}'.format (trigger.nick, args['url'], handle.id, strargs))
@@ -170,6 +172,8 @@ archiveparser = NonExitingArgumentParser (prog='a', add_help=False)
archiveparser.add_argument('--timeout', default=1*60*60, type=int, help='Maximum time for archival', metavar='SEC', choices=[60, 1*60*60, 2*60*60])
archiveparser.add_argument('--idle-timeout', default=10, type=int, help='Maximum idle seconds (i.e. no requests)', dest='idleTimeout', metavar='SEC', choices=[1, 10, 20, 30, 60])
archiveparser.add_argument('--max-body-size', default=defaultSettings.maxBodySize, type=int, dest='maxBodySize', help='Max body size', metavar='BYTES', choices=[1*1024*1024, 10*1024*1024, defaultSettings.maxBodySize, 100*1024*1024])
+archiveparser.add_argument('--concurrency', default=1, type=int, help='Parallel workers for this job', choices=range (9))
+archiveparser.add_argument('--recursive', help='Enable recursion', choices=['0', '1', '2', '3', 'prefix'])
archiveparser.add_argument('url', help='Website URL', type=isValidUrl)
@nickname_commands ('a', 'archive')
@@ -195,7 +199,8 @@ def archive (bot, trigger):
timeout=args.timeout)
args = dict (url=args.url,
enabledBehaviorNames=list (behavior.availableNames-blacklistedBehavior),
- settings=settings)
+ settings=settings, recursive=args.recursive,
+ concurrency=args.concurrency)
q = bot.memory['crocoite']['q']
q.put_nowait (('a', trigger, args))