From 048085595975b02389e8d58ed74e5a18573ee569 Mon Sep 17 00:00:00 2001 From: Lars-Dominik Braun Date: Sat, 14 Apr 2018 14:17:03 +0200 Subject: Add timeout to request body fetch When something goes wrong, these block the entire grab. --- crocoite/browser.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crocoite/browser.py b/crocoite/browser.py index 260e061..744d064 100644 --- a/crocoite/browser.py +++ b/crocoite/browser.py @@ -25,6 +25,7 @@ Chrome browser interactions. import logging from urllib.parse import urlsplit from base64 import b64decode +import pychrome class Item: """ @@ -68,7 +69,7 @@ class Item: def body (self): """ Return response body or None """ try: - body = self.tab.Network.getResponseBody (requestId=self.id) + body = self.tab.Network.getResponseBody (requestId=self.id, _timeout=60) rawBody = body['body'] base64Encoded = body['base64Encoded'] if base64Encoded: @@ -76,7 +77,7 @@ class Item: else: rawBody = rawBody.encode ('utf8') return rawBody - except pychrome.exceptions.CallMethodException: + except (pychrome.exceptions.CallMethodException, pychrome.exceptions.TimeoutException): return None def setRequest (self, req): @@ -360,7 +361,7 @@ def NullService (url): ### tests ### -import unittest, pychrome, time +import unittest, time from http.server import BaseHTTPRequestHandler class TestHTTPRequestHandler (BaseHTTPRequestHandler): -- cgit v1.2.3