diff options
author | Lars-Dominik Braun <lars@6xq.net> | 2018-03-25 13:58:47 +0200 |
---|---|---|
committer | Lars-Dominik Braun <lars@6xq.net> | 2018-03-25 14:43:25 +0200 |
commit | a21d7332e33a3e47a363004196451721d449e70b (patch) | |
tree | b6f4af4ea0838b49575a7b3ea7ab55f25fd9470d /crocoite/warc.py | |
parent | beabea4623484e7d6a9d18219efd9bf321ecc7fe (diff) | |
download | crocoite-a21d7332e33a3e47a363004196451721d449e70b.tar.gz crocoite-a21d7332e33a3e47a363004196451721d449e70b.tar.bz2 crocoite-a21d7332e33a3e47a363004196451721d449e70b.zip |
Move getResponseBody call to Item wrapper
Diffstat (limited to 'crocoite/warc.py')
-rw-r--r-- | crocoite/warc.py | 13 |
1 files changed, 2 insertions, 11 deletions
diff --git a/crocoite/warc.py b/crocoite/warc.py index e04bee4..b56d315 100644 --- a/crocoite/warc.py +++ b/crocoite/warc.py @@ -25,12 +25,10 @@ Classes writing data to WARC files import logging import json from http.server import BaseHTTPRequestHandler -from base64 import b64decode from io import BytesIO from warcio.statusandheaders import StatusAndHeaders from urllib.parse import urlsplit from logging.handlers import BufferingHandler -import pychrome from datetime import datetime from threading import Thread from queue import Queue @@ -188,15 +186,8 @@ class WarcLoader (AccountingSiteLoader): raise ValueError ('body for {} too large {} vs {}'.format (reqId, item.encodedDataLength, self.maxBodySize)) else: - try: - body = self.tab.Network.getResponseBody (requestId=reqId) - rawBody = body['body'] - base64Encoded = body['base64Encoded'] - if base64Encoded: - rawBody = b64decode (rawBody) - else: - rawBody = rawBody.encode ('utf8') - except pychrome.exceptions.CallMethodException: + rawBody = item.body + if rawBody is None: raise ValueError ('no data for {} {} {}'.format (resp['url'], resp['status'], reqId)) return rawBody, base64Encoded |