From 9d7974e3e7e8a4575ea61cb33a30fa291d12ae38 Mon Sep 17 00:00:00 2001 From: Lars-Dominik Braun Date: Thu, 3 Jan 2019 19:34:17 +0100 Subject: browser: Turn Item into RequestResponsePair MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously Item was just a simple wrapper around Chrome’s Network.* events. This turned out to be quite nasty when testing, so its replacement, RequestResponsePair, does some level of abstraction. This makes testing alot easier, since we now can simply instantiate it without building a proper DevTools event. Should come without any functional changes. --- crocoite/controller.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'crocoite/controller.py') diff --git a/crocoite/controller.py b/crocoite/controller.py index 504fa23..a64a8dc 100644 --- a/crocoite/controller.py +++ b/crocoite/controller.py @@ -30,7 +30,7 @@ from operator import attrgetter from yarl import URL from . import behavior as cbehavior -from .browser import SiteLoader, Item +from .browser import SiteLoader, RequestResponsePair from .util import getFormattedViewportMetrics, getSoftwareInfo from .behavior import ExtractLinksEvent @@ -61,13 +61,13 @@ class StatsHandler (EventHandler): self.stats = {'requests': 0, 'finished': 0, 'failed': 0, 'bytesRcv': 0} def push (self, item): - if isinstance (item, Item): + if isinstance (item, RequestResponsePair): self.stats['requests'] += 1 - if item.failed: + if not item.response: self.stats['failed'] += 1 else: self.stats['finished'] += 1 - self.stats['bytesRcv'] += item.encodedDataLength + self.stats['bytesRcv'] += item.response.bytesReceived class LogHandler (EventHandler): """ Handle items by logging information about them """ @@ -126,7 +126,7 @@ class SinglePageController: async for item in l: self.processItem (item) - async with self.service as browser, SiteLoader (browser, self.url, logger=logger) as l: + async with self.service as browser, SiteLoader (browser, logger=logger) as l: handle = asyncio.ensure_future (processQueue ()) start = time.time () @@ -153,7 +153,7 @@ class SinglePageController: } self.processItem (ControllerStart (payload)) - await l.start () + await l.navigate (self.url) for b in enabledBehavior: async for item in b.onload (): self.processItem (item) -- cgit v1.2.3