diff options
| -rw-r--r-- | contrib/celerycrocoite.py | 9 |
1 files changed, 7 insertions, 2 deletions
diff --git a/contrib/celerycrocoite.py b/contrib/celerycrocoite.py index b92e5c8..26c35ce 100644 --- a/contrib/celerycrocoite.py +++ b/contrib/celerycrocoite.py @@ -128,7 +128,7 @@ def celeryWorker (bot, q): break action, trigger, args = item if action == 'a': - handle = task.archive.delay (**args) + handle = task.controller.delay (**args) j = jobs[handle.id] = {'handle': handle, 'trigger': trigger, 'args': args} # pretty-print a few selected args @@ -136,6 +136,8 @@ def celeryWorker (bot, q): 'idleTimeout': prettyTimeDelta (args['settings']['idleTimeout']), 'timeout': prettyTimeDelta (args['settings']['timeout']), 'maxBodySize': prettyBytes (args['settings']['maxBodySize']), + 'recursive': args['recursive'], + 'concurrency': args['concurrency'], } strargs = ', '.join (map (lambda x: '{}={}'.format (*x), showargs.items ())) bot.msg (trigger.sender, '{}: {} has been queued as {} with {}'.format (trigger.nick, args['url'], handle.id, strargs)) @@ -170,6 +172,8 @@ archiveparser = NonExitingArgumentParser (prog='a', add_help=False) archiveparser.add_argument('--timeout', default=1*60*60, type=int, help='Maximum time for archival', metavar='SEC', choices=[60, 1*60*60, 2*60*60]) archiveparser.add_argument('--idle-timeout', default=10, type=int, help='Maximum idle seconds (i.e. no requests)', dest='idleTimeout', metavar='SEC', choices=[1, 10, 20, 30, 60]) archiveparser.add_argument('--max-body-size', default=defaultSettings.maxBodySize, type=int, dest='maxBodySize', help='Max body size', metavar='BYTES', choices=[1*1024*1024, 10*1024*1024, defaultSettings.maxBodySize, 100*1024*1024]) +archiveparser.add_argument('--concurrency', default=1, type=int, help='Parallel workers for this job', choices=range (9)) +archiveparser.add_argument('--recursive', help='Enable recursion', choices=['0', '1', '2', '3', 'prefix']) archiveparser.add_argument('url', help='Website URL', type=isValidUrl) @nickname_commands ('a', 'archive') @@ -195,7 +199,8 @@ def archive (bot, trigger): timeout=args.timeout) args = dict (url=args.url, enabledBehaviorNames=list (behavior.availableNames-blacklistedBehavior), - settings=settings) + settings=settings, recursive=args.recursive, + concurrency=args.concurrency) q = bot.memory['crocoite']['q'] q.put_nowait (('a', trigger, args)) |