diff options
| -rw-r--r-- | crocoite/devtools.py | 253 | ||||
| -rw-r--r-- | crocoite/test_devtools.py | 153 | ||||
| -rw-r--r-- | setup.cfg | 4 | ||||
| -rw-r--r-- | setup.py | 3 |
4 files changed, 412 insertions, 1 deletions
diff --git a/crocoite/devtools.py b/crocoite/devtools.py new file mode 100644 index 0000000..d62a8a1 --- /dev/null +++ b/crocoite/devtools.py @@ -0,0 +1,253 @@ +# Copyright (c) 2017 crocoite contributors +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +""" +Communication with Google Chrome through its DevTools protocol. +""" + +import aiohttp, websockets, json, asyncio, logging + +logger = logging.getLogger (__name__) + +class Browser: + """ + Communicate with Google Chrome through its DevTools protocol. + + Asynchronous context manager that creates a new Tab when entering. + Destroyed upon exit. + """ + + __slots__ = ('session', 'url', 'tab', 'loop') + + def __init__ (self, url, loop=None): + self.url = url + self.session = None + self.tab = None + self.loop = loop + + async def __aiter__ (self): + """ List all tabs """ + async with aiohttp.ClientSession (loop=self.loop) as session: + async with session.get ('{}/json/list'.format (self.url)) as r: + resp = await r.json () + for tab in resp: + if tab['type'] == 'page': + yield tab + + async def __aenter__ (self): + """ Create tab """ + assert self.tab is None + assert self.session is None + self.session = aiohttp.ClientSession (loop=self.loop) + async with self.session.get ('{}/json/new'.format (self.url)) as r: + resp = await r.json () + self.tab = await Tab.create (**resp) + return self.tab + + async def __aexit__ (self, *args): + assert self.tab is not None + assert self.session is not None + await self.tab.close () + async with self.session.get ('{}/json/close/{}'.format (self.url, self.tab.id)) as r: + resp = await r.text () + assert resp == 'Target is closing' + self.tab = None + await self.session.close () + self.session = None + return False + +class TabFunction: + """ + Helper class for infinite-depth tab functions. + + A method usually consists of namespace (Page, Network, …) and function name + (getFoobar) separated by a dot. This class creates these function names + while providing an intuitive Python interface (tab.Network.getFoobar). + + This was inspired by pychrome. + """ + + __slots__ = ('name', 'tab') + + def __init__ (self, name, tab): + self.name = name + self.tab = tab + + def __eq__ (self, b): + assert isinstance (b, TabFunction) + return self.name == b.name + + def __hash__ (self): + return hash (self.name) + + def __getattr__ (self, k): + return TabFunction ('{}.{}'.format (self.name, k), self.tab) + + async def __call__ (self, **kwargs): + return await self.tab (self.name, **kwargs) + + def __repr__ (self): + return '<TabFunction {}>'.format (self.name) + +class TabException (Exception): + pass + +class Crashed (TabException): + pass + +class MethodNotFound (TabException): + pass + +class InvalidParameter (TabException): + pass + +# map error codes to native exceptions +errorMap = {-32601: MethodNotFound, -32602: InvalidParameter} + +class Tab: + """ + Communicate with a single Google Chrome browser tab. + """ + __slots__ = ('id', 'wsUrl', 'ws', 'msgid', 'transactions', 'queue', '_recvHandle', 'crashed') + + def __init__ (self, tabid, ws): + """ Do not use this method, use Browser context manager. """ + self.id = tabid + self.ws = ws + self.msgid = 1 + self.crashed = False + self.transactions = {} + self.queue = asyncio.Queue () + + def __getattr__ (self, k): + return TabFunction (k, self) + + async def __call__ (self, method, **kwargs): + """ + Actually call browser method with kwargs + """ + + if self.crashed or self._recvHandle.done (): + raise Crashed () + + msgid = self.msgid + self.msgid += 1 + message = {'method': method, 'params': kwargs, 'id': msgid} + t = self.transactions[msgid] = {'event': asyncio.Event (), 'result': None} + logger.debug ('← {}'.format (message)) + await self.ws.send (json.dumps (message)) + await t['event'].wait () + ret = t['result'] + del self.transactions[msgid] + if isinstance (ret, Exception): + raise ret + return ret + + async def _recvProcess (self): + """ + Receive process that dispatches received websocket frames + + These are either events which will be put into a queue or request + responses which unblock a __call__. + """ + + async def markCrashed (reason): + # all pending requests can be considered failed since the + # browser state is lost + for v in self.transactions.values (): + v['result'] = Crashed (reason) + v['event'].set () + # and all future requests will fail as well until reloaded + self.crashed = True + await self.queue.put (Crashed (reason)) + + while True: + try: + msg = await self.ws.recv () + msg = json.loads (msg) + except Exception as e: + # right now we cannot recover from this + await markCrashed (e) + break + logger.debug ('→ {}'.format (msg)) + if 'id' in msg: + msgid = msg['id'] + t = self.transactions.get (msgid, None) + if t is not None: + if 'error' in msg: + e = msg['error'] + t['result'] = errorMap.get (e['code'], TabException) (e['code'], e['message']) + else: + t['result'] = msg['result'] + t['event'].set () + else: + # ignore stale result + pass # pragma: no cover + elif 'method' in msg: + # special treatment + if msg['method'] == 'Inspector.targetCrashed': + await markCrashed ('target') + else: + await self.queue.put (msg) + else: + assert False # pragma: no cover + + async def run (self): + self._recvHandle = asyncio.ensure_future (self._recvProcess ()) + + async def close (self): + self._recvHandle.cancel () + await self.ws.close () + # no join, throw away the queue. There will be nobody listening on the + # other end. + #await self.queue.join () + + @property + def pending (self): + return self.queue.qsize () + + async def get (self): + def getattrRecursive (obj, name): + if '.' in name: + n, ext = name.split ('.', 1) + return getattrRecursive (getattr (obj, n), ext) + else: + return getattr (obj, name) + + if self.crashed: + raise Crashed () + + ret = await self.queue.get () + if isinstance (ret, Exception): + raise ret + return getattrRecursive (self, ret['method']), ret['params'] + + @classmethod + async def create (cls, **kwargs): + """ Async init """ + # increase size limit of a single frame to something ridiciously high, + # so we can safely grab screenshots + maxSize = 100*1024*1024 # 100 MB + ws = await websockets.connect(kwargs['webSocketDebuggerUrl'], + max_size=maxSize) + ret = cls (kwargs['id'], ws) + await ret.run () + return ret + diff --git a/crocoite/test_devtools.py b/crocoite/test_devtools.py new file mode 100644 index 0000000..a6eeda2 --- /dev/null +++ b/crocoite/test_devtools.py @@ -0,0 +1,153 @@ +# Copyright (c) 2017 crocoite contributors +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import asyncio +import pytest + +from aiohttp import web +import websockets + +from .browser import ChromeService, NullService +from .devtools import Browser, Tab, MethodNotFound, Crashed, InvalidParameter + +@pytest.fixture +async def browser (): + with ChromeService () as url: + yield Browser (url) + +@pytest.fixture +async def tab (browser): + async with browser as tab: + yield tab + # make sure there are no transactions left over (i.e. no unawaited requests) + assert not tab.transactions + +async def hello(request): + return web.Response(text="Hello, world") + +@pytest.fixture +async def server (): + """ Simple HTTP server for testing notifications """ + app = web.Application() + app.add_routes([web.get('/', hello)]) + runner = web.AppRunner(app) + await runner.setup() + site = web.TCPSite(runner, 'localhost', 8080) + await site.start() + yield app + await runner.cleanup () + +@pytest.mark.asyncio +async def test_tab_create (tab): + """ Creating tabs works """ + assert isinstance (tab, Tab) + version = await tab.Browser.getVersion () + assert 'protocolVersion' in version + assert tab.pending == 0 + +@pytest.mark.asyncio +async def test_tab_close (browser): + """ Tabs are closed after using them """ + async with browser as tab: + tid = tab.id + # give the browser some time to close the tab + await asyncio.sleep (0.5) + tabs = [t['id'] async for t in browser] + assert tid not in tabs + +@pytest.mark.asyncio +async def test_tab_notify_enable_disable (tab): + """ Make sure enabling/disabling notifications works for all known namespaces """ + for name in ('Debugger', 'DOM', 'Log', 'Network', 'Page', 'Performance', 'Profiler', 'Runtime', 'Security'): + f = getattr (tab, name) + await f.enable () + await f.disable () + +@pytest.mark.asyncio +async def test_tab_unknown_method (tab): + with pytest.raises (MethodNotFound): + await tab.Nonexistent.foobar () + +@pytest.mark.asyncio +async def test_tab_invalid_argument (tab): + # should be string + with pytest.raises (InvalidParameter): + await tab.Page.captureScreenshot (format=123) + + with pytest.raises (InvalidParameter): + await tab.Page.captureScreenshot (format=[123]) + + with pytest.raises (InvalidParameter): + await tab.Page.captureScreenshot (format={123: '456'}) + +@pytest.mark.asyncio +async def test_tab_crash (tab): + with pytest.raises (Crashed): + await tab.Page.crash () + + # caling anything else now should fail as well + with pytest.raises (Crashed): + version = await tab.Browser.getVersion () + +@pytest.mark.asyncio +async def test_load (tab, server): + await tab.Network.enable () + await tab.Page.navigate (url='http://localhost:8080') + method, req = await tab.get () + assert method == tab.Network.requestWillBeSent + method, resp = await tab.get () + assert method == tab.Network.responseReceived + assert tab.pending == 0 + body = await tab.Network.getResponseBody (requestId=req['requestId']) + assert body['body'] == "Hello, world" + await tab.Network.disable () + +@pytest.mark.asyncio +async def test_recv_failure(browser): + """ Inject failure into receiver process and crash it """ + async with browser as tab: + await tab.ws.close () + with pytest.raises (Crashed): + await tab.Browser.getVersion () + + async with browser as tab: + await tab.ws.close () + with pytest.raises (Crashed): + await tab.get () + + async with browser as tab: + handle = asyncio.ensure_future (tab.get ()) + await tab.ws.close () + with pytest.raises (Crashed): + await handle + +def test_tab_function (tab): + assert tab.Network.enable.name == 'Network.enable' + assert tab.Network.disable == tab.Network.disable + assert tab.Network.enable != tab.Network.disable + assert tab.Network != tab.Network.enable + assert callable (tab.Network.enable) + assert not callable (tab.Network.enable.name) + assert 'Network.enable' in repr (tab.Network.enable) + +def test_tab_function_hash (tab): + d = {tab.Network.enable: 1, tab.Network.disable: 2, tab.Page: 3, tab.Page.enable: 4} + assert len (d) == 4 + @@ -1,2 +1,6 @@ [aliases] test=pytest +[tool:pytest] +addopts=--cov-report=html --cov=crocoite --cov-config=setup.cfg +[coverage:run] +branch=True @@ -16,6 +16,7 @@ setup( 'bottom', 'pytz', 'websockets', + 'aiohttp', ], entry_points={ 'console_scripts': [ @@ -31,5 +32,5 @@ setup( 'crocoite': ['data/*'], }, setup_requires=["pytest-runner"], - tests_require=["pytest"], + tests_require=["pytest", 'pytest-asyncio', 'pytest-cov'], ) |