From 16f12498a18119fcee6d3278aea9d7a7cfdbd7c9 Mon Sep 17 00:00:00 2001 From: Lars-Dominik Braun Date: Sat, 17 Nov 2018 10:35:49 +0100 Subject: tools: Add original HTTP header to revisit record The payloads may be the same, but the headers are usually not. --- crocoite/tools.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'crocoite/tools.py') diff --git a/crocoite/tools.py b/crocoite/tools.py index 8541ca2..da32f85 100644 --- a/crocoite/tools.py +++ b/crocoite/tools.py @@ -45,9 +45,12 @@ def mergeWarc (files, output): unique += 1 else: logging.debug ('Record {} is duplicate of {}'.format (rid, dup['id'])) + # Payload may be identical, but HTTP headers are + # (probably) not. Include them. record = writer.create_revisit_record ( headers.get_header('WARC-Target-URI'), digest=csum, - refers_to_uri=dup['uri'], refers_to_date=dup['date']) + refers_to_uri=dup['uri'], refers_to_date=dup['date'], + http_headers=record.http_headers) record.rec_headers.add_header ('WARC-Truncated', 'length') record.rec_headers.add_header ('WARC-Refers-To', dup['id']) revisit += 1 -- cgit v1.2.3