From b25c4cccafbd9572fe3e3c9c83c48c19b714a6c3 Mon Sep 17 00:00:00 2001 From: Lars-Dominik Braun Date: Sun, 29 Jul 2018 09:19:06 +0200 Subject: Add package information to warcinfo Change warcinfo record format to JSON (this is permitted by the specs) and add Python version, dependencies and their versions as well as file hashes. This should give us enough information to figure out the exact environment used to create the WARC. --- crocoite/warc.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'crocoite/warc.py') diff --git a/crocoite/warc.py b/crocoite/warc.py index a4a70ac..324d161 100644 --- a/crocoite/warc.py +++ b/crocoite/warc.py @@ -212,8 +212,12 @@ class WarcHandler (EventHandler): writer.write_record (record) def _writeControllerStart (self, item): + payload = BytesIO (json.dumps (item.payload, indent=2).encode ('utf-8')) + writer = self.writer - warcinfo = writer.create_warcinfo_record (filename=None, info=item.payload) + warcinfo = writer.create_warc_record (packageUrl ('warcinfo'), 'warcinfo', + warc_headers_dict={'Content-Type': 'text/plain; encoding=utf-8'}, + payload=payload) writer.write_record (warcinfo) def _flushLogEntries (self): -- cgit v1.2.3