From 89d5e6bcf4e3a2f6e0ed0e222e15cc80604f7351 Mon Sep 17 00:00:00 2001
From: Lars-Dominik Braun <lars@6xq.net>
Date: Thu, 1 Nov 2018 13:40:19 +0100
Subject: Add simple asyncio-based DevTool communication
Inspired by pychrome/aiochrome, but includes crash handling and async
get() instead of callbacks.
---
crocoite/devtools.py | 253 ++++++++++++++++++++++++++++++++++++++++++++++
crocoite/test_devtools.py | 153 ++++++++++++++++++++++++++++
2 files changed, 406 insertions(+)
create mode 100644 crocoite/devtools.py
create mode 100644 crocoite/test_devtools.py
(limited to 'crocoite')
diff --git a/crocoite/devtools.py b/crocoite/devtools.py
new file mode 100644
index 0000000..d62a8a1
--- /dev/null
+++ b/crocoite/devtools.py
@@ -0,0 +1,253 @@
+# Copyright (c) 2017 crocoite contributors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+"""
+Communication with Google Chrome through its DevTools protocol.
+"""
+
+import aiohttp, websockets, json, asyncio, logging
+
+logger = logging.getLogger (__name__)
+
+class Browser:
+ """
+ Communicate with Google Chrome through its DevTools protocol.
+
+ Asynchronous context manager that creates a new Tab when entering.
+ Destroyed upon exit.
+ """
+
+ __slots__ = ('session', 'url', 'tab', 'loop')
+
+ def __init__ (self, url, loop=None):
+ self.url = url
+ self.session = None
+ self.tab = None
+ self.loop = loop
+
+ async def __aiter__ (self):
+ """ List all tabs """
+ async with aiohttp.ClientSession (loop=self.loop) as session:
+ async with session.get ('{}/json/list'.format (self.url)) as r:
+ resp = await r.json ()
+ for tab in resp:
+ if tab['type'] == 'page':
+ yield tab
+
+ async def __aenter__ (self):
+ """ Create tab """
+ assert self.tab is None
+ assert self.session is None
+ self.session = aiohttp.ClientSession (loop=self.loop)
+ async with self.session.get ('{}/json/new'.format (self.url)) as r:
+ resp = await r.json ()
+ self.tab = await Tab.create (**resp)
+ return self.tab
+
+ async def __aexit__ (self, *args):
+ assert self.tab is not None
+ assert self.session is not None
+ await self.tab.close ()
+ async with self.session.get ('{}/json/close/{}'.format (self.url, self.tab.id)) as r:
+ resp = await r.text ()
+ assert resp == 'Target is closing'
+ self.tab = None
+ await self.session.close ()
+ self.session = None
+ return False
+
+class TabFunction:
+ """
+ Helper class for infinite-depth tab functions.
+
+ A method usually consists of namespace (Page, Network, …) and function name
+ (getFoobar) separated by a dot. This class creates these function names
+ while providing an intuitive Python interface (tab.Network.getFoobar).
+
+ This was inspired by pychrome.
+ """
+
+ __slots__ = ('name', 'tab')
+
+ def __init__ (self, name, tab):
+ self.name = name
+ self.tab = tab
+
+ def __eq__ (self, b):
+ assert isinstance (b, TabFunction)
+ return self.name == b.name
+
+ def __hash__ (self):
+ return hash (self.name)
+
+ def __getattr__ (self, k):
+ return TabFunction ('{}.{}'.format (self.name, k), self.tab)
+
+ async def __call__ (self, **kwargs):
+ return await self.tab (self.name, **kwargs)
+
+ def __repr__ (self):
+ return '<TabFunction {}>'.format (self.name)
+
+class TabException (Exception):
+ pass
+
+class Crashed (TabException):
+ pass
+
+class MethodNotFound (TabException):
+ pass
+
+class InvalidParameter (TabException):
+ pass
+
+# map error codes to native exceptions
+errorMap = {-32601: MethodNotFound, -32602: InvalidParameter}
+
+class Tab:
+ """
+ Communicate with a single Google Chrome browser tab.
+ """
+ __slots__ = ('id', 'wsUrl', 'ws', 'msgid', 'transactions', 'queue', '_recvHandle', 'crashed')
+
+ def __init__ (self, tabid, ws):
+ """ Do not use this method, use Browser context manager. """
+ self.id = tabid
+ self.ws = ws
+ self.msgid = 1
+ self.crashed = False
+ self.transactions = {}
+ self.queue = asyncio.Queue ()
+
+ def __getattr__ (self, k):
+ return TabFunction (k, self)
+
+ async def __call__ (self, method, **kwargs):
+ """
+ Actually call browser method with kwargs
+ """
+
+ if self.crashed or self._recvHandle.done ():
+ raise Crashed ()
+
+ msgid = self.msgid
+ self.msgid += 1
+ message = {'method': method, 'params': kwargs, 'id': msgid}
+ t = self.transactions[msgid] = {'event': asyncio.Event (), 'result': None}
+ logger.debug ('← {}'.format (message))
+ await self.ws.send (json.dumps (message))
+ await t['event'].wait ()
+ ret = t['result']
+ del self.transactions[msgid]
+ if isinstance (ret, Exception):
+ raise ret
+ return ret
+
+ async def _recvProcess (self):
+ """
+ Receive process that dispatches received websocket frames
+
+ These are either events which will be put into a queue or request
+ responses which unblock a __call__.
+ """
+
+ async def markCrashed (reason):
+ # all pending requests can be considered failed since the
+ # browser state is lost
+ for v in self.transactions.values ():
+ v['result'] = Crashed (reason)
+ v['event'].set ()
+ # and all future requests will fail as well until reloaded
+ self.crashed = True
+ await self.queue.put (Crashed (reason))
+
+ while True:
+ try:
+ msg = await self.ws.recv ()
+ msg = json.loads (msg)
+ except Exception as e:
+ # right now we cannot recover from this
+ await markCrashed (e)
+ break
+ logger.debug ('→ {}'.format (msg))
+ if 'id' in msg:
+ msgid = msg['id']
+ t = self.transactions.get (msgid, None)
+ if t is not None:
+ if 'error' in msg:
+ e = msg['error']
+ t['result'] = errorMap.get (e['code'], TabException) (e['code'], e['message'])
+ else:
+ t['result'] = msg['result']
+ t['event'].set ()
+ else:
+ # ignore stale result
+ pass # pragma: no cover
+ elif 'method' in msg:
+ # special treatment
+ if msg['method'] == 'Inspector.targetCrashed':
+ await markCrashed ('target')
+ else:
+ await self.queue.put (msg)
+ else:
+ assert False # pragma: no cover
+
+ async def run (self):
+ self._recvHandle = asyncio.ensure_future (self._recvProcess ())
+
+ async def close (self):
+ self._recvHandle.cancel ()
+ await self.ws.close ()
+ # no join, throw away the queue. There will be nobody listening on the
+ # other end.
+ #await self.queue.join ()
+
+ @property
+ def pending (self):
+ return self.queue.qsize ()
+
+ async def get (self):
+ def getattrRecursive (obj, name):
+ if '.' in name:
+ n, ext = name.split ('.', 1)
+ return getattrRecursive (getattr (obj, n), ext)
+ else:
+ return getattr (obj, name)
+
+ if self.crashed:
+ raise Crashed ()
+
+ ret = await self.queue.get ()
+ if isinstance (ret, Exception):
+ raise ret
+ return getattrRecursive (self, ret['method']), ret['params']
+
+ @classmethod
+ async def create (cls, **kwargs):
+ """ Async init """
+ # increase size limit of a single frame to something ridiciously high,
+ # so we can safely grab screenshots
+ maxSize = 100*1024*1024 # 100 MB
+ ws = await websockets.connect(kwargs['webSocketDebuggerUrl'],
+ max_size=maxSize)
+ ret = cls (kwargs['id'], ws)
+ await ret.run ()
+ return ret
+
diff --git a/crocoite/test_devtools.py b/crocoite/test_devtools.py
new file mode 100644
index 0000000..a6eeda2
--- /dev/null
+++ b/crocoite/test_devtools.py
@@ -0,0 +1,153 @@
+# Copyright (c) 2017 crocoite contributors
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+import asyncio
+import pytest
+
+from aiohttp import web
+import websockets
+
+from .browser import ChromeService, NullService
+from .devtools import Browser, Tab, MethodNotFound, Crashed, InvalidParameter
+
+@pytest.fixture
+async def browser ():
+ with ChromeService () as url:
+ yield Browser (url)
+
+@pytest.fixture
+async def tab (browser):
+ async with browser as tab:
+ yield tab
+ # make sure there are no transactions left over (i.e. no unawaited requests)
+ assert not tab.transactions
+
+async def hello(request):
+ return web.Response(text="Hello, world")
+
+@pytest.fixture
+async def server ():
+ """ Simple HTTP server for testing notifications """
+ app = web.Application()
+ app.add_routes([web.get('/', hello)])
+ runner = web.AppRunner(app)
+ await runner.setup()
+ site = web.TCPSite(runner, 'localhost', 8080)
+ await site.start()
+ yield app
+ await runner.cleanup ()
+
+@pytest.mark.asyncio
+async def test_tab_create (tab):
+ """ Creating tabs works """
+ assert isinstance (tab, Tab)
+ version = await tab.Browser.getVersion ()
+ assert 'protocolVersion' in version
+ assert tab.pending == 0
+
+@pytest.mark.asyncio
+async def test_tab_close (browser):
+ """ Tabs are closed after using them """
+ async with browser as tab:
+ tid = tab.id
+ # give the browser some time to close the tab
+ await asyncio.sleep (0.5)
+ tabs = [t['id'] async for t in browser]
+ assert tid not in tabs
+
+@pytest.mark.asyncio
+async def test_tab_notify_enable_disable (tab):
+ """ Make sure enabling/disabling notifications works for all known namespaces """
+ for name in ('Debugger', 'DOM', 'Log', 'Network', 'Page', 'Performance', 'Profiler', 'Runtime', 'Security'):
+ f = getattr (tab, name)
+ await f.enable ()
+ await f.disable ()
+
+@pytest.mark.asyncio
+async def test_tab_unknown_method (tab):
+ with pytest.raises (MethodNotFound):
+ await tab.Nonexistent.foobar ()
+
+@pytest.mark.asyncio
+async def test_tab_invalid_argument (tab):
+ # should be string
+ with pytest.raises (InvalidParameter):
+ await tab.Page.captureScreenshot (format=123)
+
+ with pytest.raises (InvalidParameter):
+ await tab.Page.captureScreenshot (format=[123])
+
+ with pytest.raises (InvalidParameter):
+ await tab.Page.captureScreenshot (format={123: '456'})
+
+@pytest.mark.asyncio
+async def test_tab_crash (tab):
+ with pytest.raises (Crashed):
+ await tab.Page.crash ()
+
+ # caling anything else now should fail as well
+ with pytest.raises (Crashed):
+ version = await tab.Browser.getVersion ()
+
+@pytest.mark.asyncio
+async def test_load (tab, server):
+ await tab.Network.enable ()
+ await tab.Page.navigate (url='http://localhost:8080')
+ method, req = await tab.get ()
+ assert method == tab.Network.requestWillBeSent
+ method, resp = await tab.get ()
+ assert method == tab.Network.responseReceived
+ assert tab.pending == 0
+ body = await tab.Network.getResponseBody (requestId=req['requestId'])
+ assert body['body'] == "Hello, world"
+ await tab.Network.disable ()
+
+@pytest.mark.asyncio
+async def test_recv_failure(browser):
+ """ Inject failure into receiver process and crash it """
+ async with browser as tab:
+ await tab.ws.close ()
+ with pytest.raises (Crashed):
+ await tab.Browser.getVersion ()
+
+ async with browser as tab:
+ await tab.ws.close ()
+ with pytest.raises (Crashed):
+ await tab.get ()
+
+ async with browser as tab:
+ handle = asyncio.ensure_future (tab.get ())
+ await tab.ws.close ()
+ with pytest.raises (Crashed):
+ await handle
+
+def test_tab_function (tab):
+ assert tab.Network.enable.name == 'Network.enable'
+ assert tab.Network.disable == tab.Network.disable
+ assert tab.Network.enable != tab.Network.disable
+ assert tab.Network != tab.Network.enable
+ assert callable (tab.Network.enable)
+ assert not callable (tab.Network.enable.name)
+ assert 'Network.enable' in repr (tab.Network.enable)
+
+def test_tab_function_hash (tab):
+ d = {tab.Network.enable: 1, tab.Network.disable: 2, tab.Page: 3, tab.Page.enable: 4}
+ assert len (d) == 4
+
--
cgit v1.2.3