diff options
author | Lars-Dominik Braun <lars@6xq.net> | 2018-11-06 16:53:16 +0100 |
---|---|---|
committer | Lars-Dominik Braun <lars@6xq.net> | 2018-11-06 16:54:58 +0100 |
commit | aeab124ac9f1e3e88f6a8ae246c90b8094e94223 (patch) | |
tree | 4b6394fcaca6b045a98f43736a6dac66a3144a7c /crocoite/cli.py | |
parent | 60fe79f2d898757f4f20aa89015e86cd63ef7871 (diff) | |
download | crocoite-aeab124ac9f1e3e88f6a8ae246c90b8094e94223.tar.gz crocoite-aeab124ac9f1e3e88f6a8ae246c90b8094e94223.tar.bz2 crocoite-aeab124ac9f1e3e88f6a8ae246c90b8094e94223.zip |
Switch single mode to asyncio
This is a direct port to asyncio without any design changes. These need
to happen in further refinements.
Fixes issue #1.
Diffstat (limited to 'crocoite/cli.py')
-rw-r--r-- | crocoite/cli.py | 13 |
1 files changed, 7 insertions, 6 deletions
diff --git a/crocoite/cli.py b/crocoite/cli.py index ca38bca..c5dee35 100644 --- a/crocoite/cli.py +++ b/crocoite/cli.py @@ -28,9 +28,10 @@ from enum import IntEnum from . import behavior from .controller import SinglePageController, defaultSettings, \ ControllerSettings, StatsHandler, LogHandler -from .browser import NullService, ChromeService, BrowserCrashed +from .browser import NullService, ChromeService from .warc import WarcHandler from .logger import Logger, JsonPrintConsumer, DatetimeConsumer, WarcHandlerConsumer +from .devtools import Crashed class SingleExitStatus(IntEnum): """ Exit status for single-shot command line """ @@ -43,7 +44,6 @@ def single (): parser.add_argument('--browser', help='DevTools URL', metavar='URL') parser.add_argument('--timeout', default=1*60*60, type=int, help='Maximum time for archival', metavar='SEC') parser.add_argument('--idle-timeout', default=30, type=int, help='Maximum idle seconds (i.e. no requests)', dest='idleTimeout', metavar='SEC') - parser.add_argument('--max-body-size', default=defaultSettings.maxBodySize, type=int, dest='maxBodySize', help='Max body size', metavar='BYTES') parser.add_argument('--behavior', help='Comma-separated list of enabled behavior scripts', dest='enabledBehaviorNames', default=list (behavior.availableMap.keys ()), @@ -59,8 +59,7 @@ def single (): service = ChromeService () if args.browser: service = NullService (args.browser) - settings = ControllerSettings (maxBodySize=args.maxBodySize, - idleTimeout=args.idleTimeout, timeout=args.timeout) + settings = ControllerSettings (idleTimeout=args.idleTimeout, timeout=args.timeout) with open (args.output, 'wb') as fd, WarcHandler (fd, logger) as warcHandler: logger.connect (WarcHandlerConsumer (warcHandler)) handler = [StatsHandler (), LogHandler (logger), warcHandler] @@ -68,9 +67,11 @@ def single (): controller = SinglePageController (args.url, fd, settings=settings, service=service, handler=handler, behavior=b, logger=logger) try: - controller.run () + loop = asyncio.get_event_loop() + loop.run_until_complete(controller.run ()) + loop.close() ret = SingleExitStatus.Ok - except BrowserCrashed: + except Crashed: ret = SingleExitStatus.BrowserCrash finally: r = handler[0].stats |