Refactoring

Reusable browser communication and WARC writing.

5 files changed, 571 insertions, 403 deletions

diff --git a/crocoite/__init__.py b/crocoite/__init__.py
index 066c83e..e23cd60 100644
--- a/crocoite/__init__.py
+++ b/crocoite/__init__.py
@@ -1,3 +1,33 @@
-#!/usr/bin/env python3
+# Copyright (c) 2017 crocoite contributors
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
 
+import os
 
+def packageData (path):
+    """
+    Locate package data, see setup.py’s data_files
+    """
+    return os.path.join (os.path.dirname (__file__), 'data', path)
+
+def packageUrl (path):
+    """
+    Create URL for package data stored into WARC
+    """
+    return 'urn:' + __package__ + ':' + path
diff --git a/crocoite/browser.py b/crocoite/browser.py
new file mode 100644
index 0000000..756fd64
--- /dev/null
+++ b/crocoite/browser.py
@@ -0,0 +1,209 @@
+# Copyright (c) 2017 crocoite contributors
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+"""
+Chrome browser interactions.
+"""
+
+import logging
+from urllib.parse import urlsplit
+
+class Item:
+    """
+    Simple wrapper containing Chrome request and response
+    """
+
+    def __init__ (self):
+        self._chromeRequest = None
+        self._chromeResponse = None
+        self.encodedDataLength = 0
+
+    def __repr__ (self):
+        return '<Item {}>'.format (self.request['url'])
+
+    @property
+    def request (self):
+        return self._chromeRequest['request']
+
+    @property
+    def response (self):
+        return self._chromeResponse['response']
+
+    @property
+    def initiator (self):
+        return self._chromeRequest['initiator']
+
+    @property
+    def id (self):
+        return self._chromeRequest['requestId']
+
+    def setRequest (self, req):
+        self._chromeRequest = req
+
+    def setResponse (self, resp):
+        self._chromeResponse = resp
+
+class SiteLoader:
+    """
+    Load site in Chrome and monitor network requests
+
+    XXX: track popup windows/new tabs and close them
+    """
+
+    allowedSchemes = {'http', 'https'}
+
+    def __init__ (self, browser, url, logger=logging.getLogger(__name__)):
+        self.requests = {}
+        self.browser = browser
+        self.url = url
+        self.logger = logger
+
+        self.tab = browser.new_tab()
+
+    def __enter__ (self):
+        tab = self.tab
+        # setup callbacks
+        tab.Network.requestWillBeSent = self._requestWillBeSent
+        tab.Network.responseReceived = self._responseReceived
+        tab.Network.loadingFinished = self._loadingFinished
+        tab.Network.loadingFailed = self._loadingFailed
+        #tab.Page.loadEventFired = loadEventFired
+
+        # start the tab
+        tab.start()
+
+        # enable events
+        tab.Network.enable()
+        tab.Page.enable ()
+        tab.Network.clearBrowserCache ()
+        if tab.Network.canClearBrowserCookies ()['result']:
+            tab.Network.clearBrowserCookies ()
+
+        return self
+
+    def __len__ (self):
+        return len (self.requests)
+
+    def start (self):
+        self.tab.Page.navigate(url=self.url)
+
+    def wait (self, timeout=1):
+        self.tab.wait (timeout)
+
+    def waitIdle (self, idleTimeout=1, maxTimeout=60):
+        step = 0
+        for i in range (0, maxTimeout):
+            self.wait (1)
+            if len (self) == 0:
+                step += 1
+                if step > idleTimeout:
+                    break
+            else:
+                step = 0
+
+    def stop (self):
+        """
+        Stop loading site
+
+        XXX: stop executing scripts
+        """
+
+        tab = self.tab
+
+        tab.Page.stopLoading ()
+        tab.Network.disable ()
+        tab.Page.disable ()
+        tab.Network.requestWillBeSent = None
+        tab.Network.responseReceived = None
+        tab.Network.loadingFinished = None
+        tab.Network.loadingFailed = None
+        tab.Page.loadEventFired = None
+
+    def __exit__ (self, exc_type, exc_value, traceback):
+        self.tab.stop ()
+        self.browser.close_tab(self.tab)
+        return False
+
+    def loadingFinished (self, item):
+        self.logger.debug ('item finished {}'.format (item))
+
+    # internal chrome callbacks
+    def _requestWillBeSent (self, **kwargs):
+        reqId = kwargs['requestId']
+        req = kwargs['request']
+
+        url = urlsplit (req['url'])
+        if url.scheme not in self.allowedSchemes:
+            return
+
+        item = self.requests.get (reqId)
+        if item:
+            # redirects never “finish” loading, but yield another requestWillBeSent with this key set
+            redirectResp = kwargs.get ('redirectResponse')
+            if redirectResp:
+                resp = {'requestId': reqId, 'response': redirectResp}
+                item.setResponse (resp)
+                self.loadingFinished (item, redirect=True)
+                self.logger.debug ('redirected request {} has url {}'.format (reqId, req['url']))
+            else:
+                self.logger.warn ('request {} already exists, overwriting.'.format (reqId))
+
+        item = Item ()
+        item.setRequest (kwargs)
+        self.requests[reqId] = item
+
+    def _responseReceived (self, **kwargs):
+        reqId = kwargs['requestId']
+        item = self.requests.get (reqId)
+        if item is None:
+            return
+
+        resp = kwargs['response']
+        url = urlsplit (resp['url'])
+        if url.scheme in self.allowedSchemes:
+            self.logger.debug ('response {} {}'.format (reqId, resp['url']))
+            item.setResponse (kwargs)
+        else:
+            self.logger.warn ('response: ignoring scheme {}'.format (url.scheme))
+
+    def _loadingFinished (self, **kwargs):
+        """
+        Item was fully loaded. For some items the request body is not available
+        when responseReceived is fired, thus move everything here.
+        """
+        reqId = kwargs['requestId']
+        item = self.requests.pop (reqId, None)
+        if item is None:
+            # we never recorded this request (blacklisted scheme, for example)
+            return
+        req = item.request
+        resp = item.response
+        assert req['url'] == resp['url'], 'req and resp urls are not the same {} vs {}'.format (req['url'], resp['url'])
+        url = urlsplit (resp['url'])
+        if url.scheme in self.allowedSchemes:
+            self.logger.debug ('finished {} {}'.format (reqId, req['url']))
+            item.encodedDataLength = kwargs['encodedDataLength']
+            self.loadingFinished (item)
+
+    def _loadingFailed (self, **kwargs):
+        reqId = kwargs['requestId']
+        self.logger.debug ('failed {} {}'.format (reqId, kwargs['errorText'], kwargs.get ('blockedReason')))
+        item = self.requests.pop (reqId, None)
+
diff --git a/crocoite/cli.py b/crocoite/cli.py
index 640d207..a97f0ce 100644
--- a/crocoite/cli.py
+++ b/crocoite/cli.py
@@ -18,327 +18,23 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-import os, sys, json
-import pychrome
-from urllib.parse import urlsplit
-from warcio.warcwriter import WARCWriter
-from warcio.statusandheaders import StatusAndHeaders
-from base64 import b64decode
-import logging
-from logging.handlers import BufferingHandler
-from http.server import BaseHTTPRequestHandler
-from io import BytesIO
-import argparse
-import tempfile, random
-
-from html5lib.treewalkers.base import TreeWalker
-from html5lib.filters.base import Filter
-from html5lib.serializer import HTMLSerializer
-from html5lib import constants
-
-from . import html
-
-logger = logging.getLogger(__name__)
-
-# 10 MB, encoded! (i.e. actual data can be larger due to compression)
-maxBodySize = 10*1024*1024
-
-def packageData (path):
-    """
-    Locate package data, see setup.py’s data_files
-    """
-    return os.path.join (os.path.dirname (__file__), 'data', path)
-
-def packageUrl (path):
-    """
-    Create URL for package data stored into WARC
-    """
-    return 'urn:' + __package__ + ':' + path
-
-def randomString (length=None, chars='abcdefghijklmnopqrstuvwxyz'):
-    if length is None:
-        length = random.randint (16, 32)
-    return ''.join (map (lambda x: random.choice (chars), range (length)))
-
-class WARCLogHandler (BufferingHandler):
-    """
-    Buffered log handler, flushing to warcio
-    """
-
-    contentType = 'text/plain; charset=utf-8'
-
-    def __init__ (self, capacity, warcfile):
-        BufferingHandler.__init__ (self, capacity)
-        self.warcfile = warcfile
-
-    def flush (self):
-        self.acquire ()
-        try:
-            buf = ''
-            for record in self.buffer:
-                buf += self.format (record)
-                buf += '\n'
-            # XXX: record type?
-            record = self.warcfile.create_warc_record (
-                    packageUrl ('log'), 'metadata',
-                    payload=BytesIO (buf.encode ('utf8')),
-                    warc_headers_dict={'Content-Type': self.contentType})
-            self.warcfile.write_record(record)
-            self.buffer = []
-        finally:
-            self.release ()
-
-class ChromeTreeWalker (TreeWalker):
-    """
-    Recursive html5lib TreeWalker for Google Chrome method DOM.getDocument
-    """
-
-    def recurse (self, node):
-        name = node['nodeName']
-        if name.startswith ('#'):
-            if name == '#text':
-                yield from self.text (node['nodeValue'])
-            elif name == '#comment':
-                yield self.comment (node['nodeValue'])
-            elif name == '#document':
-                for child in node.get ('children', []):
-                    yield from self.recurse (child)
-            else:
-                assert False, name
-        else:
-            default_namespace = constants.namespaces["html"]
-
-            attributes = node.get ('attributes', [])
-            convertedAttr = {}
-            for i in range (0, len (attributes), 2):
-                convertedAttr[(default_namespace, attributes[i])] = attributes[i+1]
-
-            children = node.get ('children', [])
-            if name.lower() in html.voidTags and not children:
-                yield from self.emptyTag (default_namespace, name, convertedAttr)
-            else:
-                yield self.startTag (default_namespace, name, convertedAttr)
-                for child in node.get ('children', []):
-                    yield from self.recurse (child)
-                yield self.endTag ('', name)
-
-    def __iter__ (self):
-        assert self.tree['nodeName'] == '#document'
-        return self.recurse (self.tree)
-
-    def split (self):
-        """
-        Split response returned by DOM.getDocument(pierce=True) into independent documents
-        """
-        def recurse (node):
-            contentDocument = node.get ('contentDocument')
-            if contentDocument:
-                assert contentDocument['nodeName'] == '#document'
-                yield contentDocument
-                yield from recurse (contentDocument)
-
-            for child in node.get ('children', []):
-                yield from recurse (child)
-
-        if self.tree['nodeName'] == '#document':
-            yield self.tree
-        yield from recurse (self.tree)
-
-class StripTagFilter (Filter):
-    """
-    Remove arbitrary tags
-    """
-
-    def __init__ (self, source, tags):
-        Filter.__init__ (self, source)
-        self.tags = set (map (str.lower, tags))
-
-    def __iter__(self):
-        delete = 0
-        for token in Filter.__iter__(self):
-            tokenType = token['type']
-            if tokenType in {'StartTag', 'EmptyTag'}:
-                if delete > 0 or token['name'].lower () in self.tags:
-                    delete += 1
-            if delete == 0:
-                yield token
-            if tokenType == 'EndTag' and delete > 0:
-                delete -= 1
-
-class StripAttributeFilter (Filter):
-    """
-    Remove arbitrary HTML attributes
-    """
-
-    def __init__ (self, source, attributes):
-        Filter.__init__ (self, source)
-        self.attributes = set (map (str.lower, attributes))
-
-    def __iter__(self):
-        default_namespace = constants.namespaces["html"]
-        for token in Filter.__iter__(self):
-            data = token.get ('data')
-            if data and token['type'] in {'StartTag', 'EmptyTag'}:
-                newdata = {}
-                for (namespace, k), v in data.items ():
-                    if k.lower () not in self.attributes:
-                        newdata[(namespace, k)] = v
-                token['data'] = newdata
-            yield token
-
 def main ():
-    def getStatusText (response):
-        text = response.get ('statusText')
-        if text:
-            return text
-        text = BaseHTTPRequestHandler.responses.get (response['status'])
-        if text:
-            return text[0]
-        return 'No status text available'
-
-    def requestWillBeSent (**kwargs):
-        req = kwargs.get ('request')
-        reqId = kwargs['requestId']
-        url = urlsplit (req['url'])
-        if url.scheme in ('http', 'https'):
-            logger.debug ('sending {} {}'.format (reqId, req['url']))
-            if reqId in requests:
-                redirectResp = kwargs.get ('redirectResponse')
-                if redirectResp:
-                    requests[reqId]['response'] = redirectResp
-                    # XXX: can we retrieve the original response body right now?
-                    itemToWarc (reqId, requests[reqId], ignoreBody=True)
-                else:
-                    logger.warn ('request {} already exists, overwriting.'.format (reqId))
-            requests[reqId] = {}
-            requests[reqId]['request'] = req
-            requests[reqId]['initiator'] = kwargs['initiator']
-        else:
-            logger.warn ('request: ignoring scheme {}'.format (url.scheme))
-
-    def responseReceived (**kwargs):
-        resp = kwargs['response']
-        reqId = kwargs['requestId']
-        url = urlsplit (resp['url'])
-        if url.scheme in ('http', 'https') and reqId in requests:
-            logger.debug ('response {} {}'.format (reqId, resp['url']))
-            requests[reqId]['response'] = resp
-        else:
-            logger.warn ('response: ignoring scheme {}'.format (url.scheme))
-
-    def loadEventFired (**kwargs):
-        """
-        Equivalent to DOM ready JavaScript event
-        """
-        root = tab.DOM.getDocument ()
-        rootId = root['root']['nodeId']
-        links = tab.DOM.querySelectorAll (nodeId=rootId, selector='a')
-        #for i in links['nodeIds']:
-        #    print ('link', tab.DOM.getAttributes (nodeId=i))
-
-    def loadingFinished (**kwargs):
-        """
-        Item was fully loaded. For some items the request body is not available
-        when responseReceived is fired, thus move everything here.
-        """
-        reqId = kwargs['requestId']
-        # we never recorded this request
-        if reqId not in requests:
-            return
-        item = requests[reqId]
-        req = item['request']
-        resp = item['response']
-        url = urlsplit (resp['url'])
-        if url.scheme in ('http', 'https'):
-            logger.debug ('finished {} {}'.format (reqId, req['url']))
-            itemToWarc (reqId, item, kwargs['encodedDataLength'])
-            del requests[reqId]
-
-    def itemToWarc (reqId, item, encodedDataLength=0, ignoreBody=False):
-        req = item['request']
-        resp = item['response']
-        url = urlsplit (resp['url'])
-
-        # overwrite request headers with those actually sent
-        newReqHeaders = resp.get ('requestHeaders')
-        if newReqHeaders:
-            req['headers'] = newReqHeaders
-
-        postData = req.get ('postData')
-        if postData:
-            postData = BytesIO (postData.encode ('utf8'))
-        path = url.path
-        if url.query:
-            path += '?' + url.query
-        httpHeaders = StatusAndHeaders('{} {} HTTP/1.1'.format (req['method'], path),
-                req['headers'], protocol='HTTP/1.1', is_http_request=True)
-        initiator = item['initiator']
-        warcHeaders = {
-                'X-Chrome-Initiator': json.dumps (initiator),
-                }
-        record = writer.create_warc_record(req['url'], 'request',
-                payload=postData, http_headers=httpHeaders,
-                warc_headers_dict=warcHeaders)
-        writer.write_record(record)
-        concurrentTo = record.rec_headers['WARC-Record-ID']
-
-        # check body size first, since we’re loading everything into memory
-        if encodedDataLength < maxBodySize:
-            try:
-                if ignoreBody:
-                    rawBody = b''
-                    base64Encoded = True
-                else:
-                    body = tab.Network.getResponseBody (requestId=reqId)
-                    rawBody = body['body']
-                    base64Encoded = body['base64Encoded']
-                    if base64Encoded:
-                        rawBody = b64decode (rawBody)
-                    else:
-                        rawBody = rawBody.encode ('utf8')
-
-                httpHeaders = StatusAndHeaders('{} {}'.format (resp['status'],
-                        getStatusText (resp)), resp['headers'], protocol='HTTP/1.1')
-
-                # Content is saved decompressed and decoded, remove these headers
-                blacklistedHeaders = {'transfer-encoding', 'content-encoding'}
-                for h in blacklistedHeaders:
-                    httpHeaders.remove_header (h)
-
-                # chrome sends nothing but utf8 encoded text. Fortunately HTTP
-                # headers take precedence over the document’s <meta>, thus we can
-                # easily override those.
-                contentType = resp['mimeType']
-                if not base64Encoded:
-                    contentType += '; charset=utf-8'
-                httpHeaders.replace_header ('content-type', contentType)
-
-                httpHeaders.replace_header ('content-length', '{:d}'.format (len (rawBody)))
-
-                warcHeaders = {
-                        'WARC-Concurrent-To': concurrentTo,
-                        'WARC-IP-Address': resp.get ('remoteIPAddress', ''),
-                        'X-Chrome-Protocol': resp.get ('protocol', ''),
-                        'X-Chrome-FromDiskCache': str (resp.get ('fromDiskCache')),
-                        'X-Chrome-ConnectionReused': str (resp.get ('connectionReused')),
-                        'X-Chrome-Base64Body': str (base64Encoded),
-                        }
-                record = writer.create_warc_record(resp['url'], 'response',
-                        warc_headers_dict=warcHeaders, payload=BytesIO (rawBody),
-                        http_headers=httpHeaders)
-                writer.write_record(record)
-            except pychrome.exceptions.CallMethodException:
-                logger.error ('no data for {} {} {}'.format (resp['url'],
-                        resp['status'], reqId))
-        else:
-            logger.warn ('body for {} is too large, {} bytes'.format (resp['url'], kwargs['encodedDataLength']))
-
-    def loadingFailed (**kwargs):
-        reqId = kwargs['requestId']
-        logger.debug ('failed {} {}'.format (reqId, kwargs['errorText'], kwargs.get ('blockedReason')))
-        if reqId in requests:
-            del requests[reqId]
-
+    import os, random, logging, argparse
+    from io import BytesIO
+    import pychrome
+    from urllib.parse import urlsplit
+    from warcio.warcwriter import WARCWriter
+    from warcio.statusandheaders import StatusAndHeaders
+    from html5lib.serializer import HTMLSerializer
+    from . import html, packageData, packageUrl
+    from .warc import WarcLoader
+    from .html import StripAttributeFilter, StripTagFilter, ChromeTreeWalker
+
+    def randomString (length=None, chars='abcdefghijklmnopqrstuvwxyz'):
+        if length is None:
+            length = random.randint (16, 32)
+        return ''.join (map (lambda x: random.choice (chars), range (length)))
+    
     def getFormattedViewportMetrics (tab):
         layoutMetrics = tab.Page.getLayoutMetrics ()
         # XXX: I’m not entirely sure which one we should use here
@@ -384,7 +80,7 @@ def main ():
                                 'X-Chrome-Viewport': viewport})
                 writer.write_record (record)
 
-    def emulateScreenMetrics (tab):
+    def emulateScreenMetrics (l):
         """
         Emulate different screen sizes, causing the site to fetch assets (img
         srcset and css, for example) for different screen resolutions.
@@ -404,13 +100,12 @@ def main ():
                 {'width': 1920, 'height': 1080, 'deviceScaleFactor': 1, 'mobile': False},
                 ]
         for s in sizes:
-            tab.Emulation.setDeviceMetricsOverride (**s)
-            tab.wait (1)
+            l.tab.Emulation.setDeviceMetricsOverride (**s)
+            l.wait (1)
         # XXX: this seems to be broken, it does not clear the override
         #tab.Emulation.clearDeviceMetricsOverride ()
         # wait until assets finished loading
-        while len (requests) != 0:
-            tab.wait (1)
+        l.waitIdle (2, 10)
 
     def loadScripts (paths, scripts=[]):
         for p in paths:
@@ -427,6 +122,7 @@ def main ():
                 warc_headers_dict={'Content-Type': 'application/javascript; charset=utf-8'})
         writer.write_record (record)
 
+    logger = logging.getLogger(__name__)
     logging.basicConfig (level=logging.DEBUG)
 
     parser = argparse.ArgumentParser(description='Save website to WARC using Google Chrome.')
@@ -434,7 +130,8 @@ def main ():
     parser.add_argument('--timeout', default=10, type=int, help='Maximum time for archival')
     parser.add_argument('--idle-timeout', default=2, type=int, help='Maximum idle seconds (i.e. no requests)', dest='idleTimeout')
     parser.add_argument('--log-buffer', default=1000, type=int, dest='logBuffer')
-    parser.add_argument('--keep-tab', action='store_true', default=False, dest='keepTab', help='Keep tab open')
+    parser.add_argument('--max-body-size', default=10*1024*1024, type=int, dest='maxBodySize', help='Max body size in bytes')
+    #parser.add_argument('--keep-tab', action='store_true', default=False, dest='keepTab', help='Keep tab open')
     parser.add_argument('--onload', default=[], action='append', help='Inject JavaScript file before loading page')
     parser.add_argument('--onsnapshot', default=[], action='append', help='Run JavaScript files before creating DOM snapshot')
     parser.add_argument('url', help='Website URL')
@@ -448,93 +145,44 @@ def main ():
     onload = loadScripts (args.onload, ['var {} = false;\n'.format (stopVarname)]).replace (stopVarname, newStopVarname)
     stopVarname = newStopVarname
 
-    # temporary store for requests
-    requests = {}
-
-    # create a browser instance
     browser = pychrome.Browser(url=args.browser)
 
-    # create a tab
-    tab = browser.new_tab()
-
-    # setup callbacks
-    tab.Network.requestWillBeSent = requestWillBeSent
-    tab.Network.responseReceived = responseReceived
-    tab.Network.loadingFinished = loadingFinished
-    tab.Network.loadingFailed = loadingFailed
-    tab.Page.loadEventFired = loadEventFired
-
-    # start the tab
-    tab.start()
-
     fd = open (args.output, 'wb')
     writer = WARCWriter (fd, gzip=True)
-    version = tab.Browser.getVersion ()
-    payload = {
-            'software': __package__,
-            'browser': version['product'],
-            'useragent': version['userAgent'],
-            'viewport': getFormattedViewportMetrics (tab),
-            }
-    warcinfo = writer.create_warcinfo_record (filename=None, info=payload)
-    writer.write_record (warcinfo)
-
-    warcLogger = WARCLogHandler (args.logBuffer, writer)
-    logger.addHandler (warcLogger)
-
-    # save onload script
-    writeScript ('onload', onload, writer)
 
-    # enable events
-    tab.Network.enable()
-    tab.Page.enable ()
-    tab.Network.clearBrowserCache ()
-    if tab.Network.canClearBrowserCookies ()['result']:
-        tab.Network.clearBrowserCookies ()
-    # inject our custom javascript to the page before loading
-    tab.Page.addScriptToEvaluateOnNewDocument (source=onload)
-
-    tab.Page.navigate(url=args.url)
-
-    idleTimeout = 0
-    for i in range (0, args.timeout):
-        tab.wait (1)
-        if len (requests) == 0:
-            idleTimeout += 1
-            if idleTimeout > args.idleTimeout:
-                break
-        else:
-            idleTimeout = 0
+    with WarcLoader (browser, args.url, writer, logBuffer=args.logBuffer,
+            maxBodySize=args.maxBodySize) as l:
+        version = l.tab.Browser.getVersion ()
+        payload = {
+                'software': __package__,
+                'browser': version['product'],
+                'useragent': version['userAgent'],
+                'viewport': getFormattedViewportMetrics (l.tab),
+                }
+        warcinfo = writer.create_warcinfo_record (filename=None, info=payload)
+        writer.write_record (warcinfo)
+        # save onload script as well
+        writeScript ('onload', onload, writer)
 
-    # get ready for snapshot: stop loading and scripts, disable events
-    tab.Runtime.evaluate (expression='{} = true; window.scrollTo (0, 0);'.format (stopVarname), returnByValue=True)
-    # if we stopped due to timeout, wait for remaining assets
-    while len (requests) != 0:
-        tab.wait (1)
+        # inject our custom javascript to the page before loading
+        l.tab.Page.addScriptToEvaluateOnNewDocument (source=onload)
+        l.start ()
 
-    emulateScreenMetrics (tab)
+        l.waitIdle (args.idleTimeout, args.timeout)
 
-    tab.Page.stopLoading ()
-    tab.Network.disable ()
-    tab.Page.disable ()
-    tab.Network.requestWillBeSent = None
-    tab.Network.responseReceived = None
-    tab.Network.loadingFinished = None
-    tab.Network.loadingFailed = None
-    tab.Page.loadEventFired = None
+        # get ready for snapshot: stop loading and scripts, disable events
+        l.tab.Runtime.evaluate (expression='{} = true; window.scrollTo (0, 0);'.format (stopVarname), returnByValue=True)
+        # if we stopped due to timeout, wait for remaining assets
+        l.waitIdle (2, 10)
 
-    script = loadScripts (args.onsnapshot)
-    writeScript ('onsnapshot', script, writer)
-    tab.Runtime.evaluate (expression=script, returnByValue=True)
-    writeDOMSnapshot (tab, writer)
+        emulateScreenMetrics (l)
 
-    tab.stop()
-    if not args.keepTab:
-        browser.close_tab(tab)
+        l.stop ()
 
-    logger.removeHandler (warcLogger)
-    warcLogger.flush ()
-    fd.close ()
+        script = loadScripts (args.onsnapshot)
+        writeScript ('onsnapshot', script, writer)
+        l.tab.Runtime.evaluate (expression=script, returnByValue=True)
+        writeDOMSnapshot (l.tab, writer)
 
     return True
 
diff --git a/crocoite/html.py b/crocoite/html.py
index 34fe26b..f891101 100644
--- a/crocoite/html.py
+++ b/crocoite/html.py
@@ -18,6 +18,10 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+"""
+HTML helper
+"""
+
 # HTML void tags, see https://html.spec.whatwg.org/multipage/syntax.html#void-elements
 voidTags = {'area',
         'base',
@@ -99,3 +103,106 @@ eventAttributes = {'onabort',
         'onvolumechange',
         'onwaiting'}
 
+from html5lib.treewalkers.base import TreeWalker
+from html5lib.filters.base import Filter
+from html5lib.serializer import HTMLSerializer
+from html5lib import constants
+
+class ChromeTreeWalker (TreeWalker):
+    """
+    Recursive html5lib TreeWalker for Google Chrome method DOM.getDocument
+    """
+
+    def recurse (self, node):
+        name = node['nodeName']
+        if name.startswith ('#'):
+            if name == '#text':
+                yield from self.text (node['nodeValue'])
+            elif name == '#comment':
+                yield self.comment (node['nodeValue'])
+            elif name == '#document':
+                for child in node.get ('children', []):
+                    yield from self.recurse (child)
+            else:
+                assert False, name
+        else:
+            default_namespace = constants.namespaces["html"]
+
+            attributes = node.get ('attributes', [])
+            convertedAttr = {}
+            for i in range (0, len (attributes), 2):
+                convertedAttr[(default_namespace, attributes[i])] = attributes[i+1]
+
+            children = node.get ('children', [])
+            if name.lower() in voidTags and not children:
+                yield from self.emptyTag (default_namespace, name, convertedAttr)
+            else:
+                yield self.startTag (default_namespace, name, convertedAttr)
+                for child in node.get ('children', []):
+                    yield from self.recurse (child)
+                yield self.endTag ('', name)
+
+    def __iter__ (self):
+        assert self.tree['nodeName'] == '#document'
+        return self.recurse (self.tree)
+
+    def split (self):
+        """
+        Split response returned by DOM.getDocument(pierce=True) into independent documents
+        """
+        def recurse (node):
+            contentDocument = node.get ('contentDocument')
+            if contentDocument:
+                assert contentDocument['nodeName'] == '#document'
+                yield contentDocument
+                yield from recurse (contentDocument)
+
+            for child in node.get ('children', []):
+                yield from recurse (child)
+
+        if self.tree['nodeName'] == '#document':
+            yield self.tree
+        yield from recurse (self.tree)
+
+class StripTagFilter (Filter):
+    """
+    Remove arbitrary tags
+    """
+
+    def __init__ (self, source, tags):
+        Filter.__init__ (self, source)
+        self.tags = set (map (str.lower, tags))
+
+    def __iter__(self):
+        delete = 0
+        for token in Filter.__iter__(self):
+            tokenType = token['type']
+            if tokenType in {'StartTag', 'EmptyTag'}:
+                if delete > 0 or token['name'].lower () in self.tags:
+                    delete += 1
+            if delete == 0:
+                yield token
+            if tokenType == 'EndTag' and delete > 0:
+                delete -= 1
+
+class StripAttributeFilter (Filter):
+    """
+    Remove arbitrary HTML attributes
+    """
+
+    def __init__ (self, source, attributes):
+        Filter.__init__ (self, source)
+        self.attributes = set (map (str.lower, attributes))
+
+    def __iter__(self):
+        default_namespace = constants.namespaces["html"]
+        for token in Filter.__iter__(self):
+            data = token.get ('data')
+            if data and token['type'] in {'StartTag', 'EmptyTag'}:
+                newdata = {}
+                for (namespace, k), v in data.items ():
+                    if k.lower () not in self.attributes:
+                        newdata[(namespace, k)] = v
+                token['data'] = newdata
+            yield token
+
diff --git a/crocoite/warc.py b/crocoite/warc.py
new file mode 100644
index 0000000..e06b1c7
--- /dev/null
+++ b/crocoite/warc.py
@@ -0,0 +1,174 @@
+# Copyright (c) 2017 crocoite contributors
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# 
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+# 
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+"""
+Classes writing data to WARC files
+"""
+
+import logging
+import json
+from .browser import SiteLoader
+from . import packageUrl
+from http.server import BaseHTTPRequestHandler
+from base64 import b64decode
+from io import BytesIO
+from warcio.statusandheaders import StatusAndHeaders
+from urllib.parse import urlsplit
+from logging.handlers import BufferingHandler
+import pychrome
+
+class WARCLogHandler (BufferingHandler):
+    """
+    Buffered log handler, flushing to warcio
+    """
+
+    contentType = 'text/plain; charset=utf-8'
+
+    def __init__ (self, capacity, warcfile):
+        BufferingHandler.__init__ (self, capacity)
+        self.warcfile = warcfile
+
+    def flush (self):
+        self.acquire ()
+        try:
+            if self.buffer:
+                buf = ''
+                for record in self.buffer:
+                    buf += self.format (record)
+                    buf += '\n'
+                # XXX: record type?
+                record = self.warcfile.create_warc_record (
+                        packageUrl ('log'), 'metadata',
+                        payload=BytesIO (buf.encode ('utf8')),
+                        warc_headers_dict={'Content-Type': self.contentType})
+                self.warcfile.write_record(record)
+                self.buffer = []
+        finally:
+            self.release ()
+
+class WarcLoader (SiteLoader):
+    def __init__ (self, browser, url, writer,
+            logger=logging.getLogger(__name__), logBuffer=1000,
+            maxBodySize=10*1024*1024):
+        SiteLoader.__init__ (self, browser, url, logger)
+        self.writer = writer
+        self.maxBodySize = maxBodySize
+        self.warcLogger = WARCLogHandler (logBuffer, writer)
+        self.logger.addHandler (self.warcLogger)
+
+    def __exit__ (self, exc_type, exc_value, traceback):
+        self.logger.removeHandler (self.warcLogger)
+        self.warcLogger.flush ()
+        return SiteLoader.__exit__ (self, exc_type, exc_value, traceback)
+
+    @staticmethod
+    def getStatusText (response):
+        text = response.get ('statusText')
+        if text:
+            return text
+        text = BaseHTTPRequestHandler.responses.get (response['status'])
+        if text:
+            return text[0]
+        return 'No status text available'
+
+    def loadingFinished (self, item, redirect=False):
+        writer = self.writer
+
+        req = item.request
+        reqId = item.id
+        resp = item.response
+        url = urlsplit (resp['url'])
+
+        # overwrite request headers with those actually sent
+        newReqHeaders = resp.get ('requestHeaders')
+        if newReqHeaders:
+            req['headers'] = newReqHeaders
+
+        postData = req.get ('postData')
+        if postData:
+            postData = BytesIO (postData.encode ('utf8'))
+        path = url.path
+        if url.query:
+            path += '?' + url.query
+        httpHeaders = StatusAndHeaders('{} {} HTTP/1.1'.format (req['method'], path),
+                req['headers'], protocol='HTTP/1.1', is_http_request=True)
+        initiator = item.initiator
+        warcHeaders = {
+                'X-Chrome-Initiator': json.dumps (initiator),
+                }
+        record = writer.create_warc_record(req['url'], 'request',
+                payload=postData, http_headers=httpHeaders,
+                warc_headers_dict=warcHeaders)
+        writer.write_record(record)
+        concurrentTo = record.rec_headers['WARC-Record-ID']
+
+        # now the response
+        warcHeaders = {
+                'WARC-Concurrent-To': concurrentTo,
+                'WARC-IP-Address': resp.get ('remoteIPAddress', ''),
+                'X-Chrome-Protocol': resp.get ('protocol', ''),
+                'X-Chrome-FromDiskCache': str (resp.get ('fromDiskCache')),
+                'X-Chrome-ConnectionReused': str (resp.get ('connectionReused')),
+                }
+
+        rawBody = b''
+        base64Encoded = False
+        try:
+            # check body size first, since we’re loading everything into memory
+            if item.encodedDataLength < self.maxBodySize:
+                body = self.tab.Network.getResponseBody (requestId=reqId)
+                rawBody = body['body']
+                base64Encoded = body['base64Encoded']
+                if base64Encoded:
+                    rawBody = b64decode (rawBody)
+                    warcHeaders['X-Chrome-Base64Body'] = str (True)
+                else:
+                    rawBody = rawBody.encode ('utf8')
+            else:
+                self.logger.error ('body for {} too large {} vs {}'.format (reqId,
+                        item.encodedDataLength, self.maxBodySize))
+        except pychrome.exceptions.CallMethodException:
+            self.logger.error ('no data for {} {} {}'.format (resp['url'],
+                    resp['status'], reqId))
+
+        httpHeaders = StatusAndHeaders('{} {}'.format (resp['status'],
+                self.getStatusText (resp)), resp['headers'], protocol='HTTP/1.1')
+
+        # Content is saved decompressed and decoded, remove these headers
+        blacklistedHeaders = {'transfer-encoding', 'content-encoding'}
+        for h in blacklistedHeaders:
+            httpHeaders.remove_header (h)
+
+        # chrome sends nothing but utf8 encoded text. Fortunately HTTP
+        # headers take precedence over the document’s <meta>, thus we can
+        # easily override those.
+        contentType = resp.get ('mimeType')
+        if contentType:
+            if not base64Encoded:
+                contentType += '; charset=utf-8'
+            httpHeaders.replace_header ('content-type', contentType)
+
+        httpHeaders.replace_header ('content-length', '{:d}'.format (len (rawBody)))
+
+        record = writer.create_warc_record(resp['url'], 'response',
+                warc_headers_dict=warcHeaders, payload=BytesIO (rawBody),
+                http_headers=httpHeaders)
+        writer.write_record(record)
+