summaryrefslogtreecommitdiff
path: root/crocoite/warc.py
diff options
context:
space:
mode:
authorLars-Dominik Braun <lars@6xq.net>2018-03-25 13:58:47 +0200
committerLars-Dominik Braun <lars@6xq.net>2018-03-25 14:43:25 +0200
commita21d7332e33a3e47a363004196451721d449e70b (patch)
treeb6f4af4ea0838b49575a7b3ea7ab55f25fd9470d /crocoite/warc.py
parentbeabea4623484e7d6a9d18219efd9bf321ecc7fe (diff)
downloadcrocoite-a21d7332e33a3e47a363004196451721d449e70b.tar.gz
crocoite-a21d7332e33a3e47a363004196451721d449e70b.tar.bz2
crocoite-a21d7332e33a3e47a363004196451721d449e70b.zip
Move getResponseBody call to Item wrapper
Diffstat (limited to 'crocoite/warc.py')
-rw-r--r--crocoite/warc.py13
1 files changed, 2 insertions, 11 deletions
diff --git a/crocoite/warc.py b/crocoite/warc.py
index e04bee4..b56d315 100644
--- a/crocoite/warc.py
+++ b/crocoite/warc.py
@@ -25,12 +25,10 @@ Classes writing data to WARC files
import logging
import json
from http.server import BaseHTTPRequestHandler
-from base64 import b64decode
from io import BytesIO
from warcio.statusandheaders import StatusAndHeaders
from urllib.parse import urlsplit
from logging.handlers import BufferingHandler
-import pychrome
from datetime import datetime
from threading import Thread
from queue import Queue
@@ -188,15 +186,8 @@ class WarcLoader (AccountingSiteLoader):
raise ValueError ('body for {} too large {} vs {}'.format (reqId,
item.encodedDataLength, self.maxBodySize))
else:
- try:
- body = self.tab.Network.getResponseBody (requestId=reqId)
- rawBody = body['body']
- base64Encoded = body['base64Encoded']
- if base64Encoded:
- rawBody = b64decode (rawBody)
- else:
- rawBody = rawBody.encode ('utf8')
- except pychrome.exceptions.CallMethodException:
+ rawBody = item.body
+ if rawBody is None:
raise ValueError ('no data for {} {} {}'.format (resp['url'],
resp['status'], reqId))
return rawBody, base64Encoded