From 9ff793e96139ed40090ab9d8c3cae99b284858e5 Mon Sep 17 00:00:00 2001 From: Lars-Dominik Braun Date: Tue, 2 Jul 2019 09:14:55 +0200 Subject: Stabilize WARC headers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In preparation for 1.0 release: - Correct mime types - Add X-Crocoite-Type, so logs, scripts, dom-snapshots and screenshots can be identified easily - Remove random WARC headers like X-Chrome-Initiator. We don’t want to maintain those. - Remove non-standard urn-based package URLs. Can’t use them without a urn-registration --- crocoite/util.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'crocoite/util.py') diff --git a/crocoite/util.py b/crocoite/util.py index ded5e99..da377a3 100644 --- a/crocoite/util.py +++ b/crocoite/util.py @@ -26,6 +26,8 @@ import random, sys, platform, os, json, urllib from datetime import datetime import hashlib, pkg_resources +from yarl import URL + class StrJsonEncoder (json.JSONEncoder): """ JSON encoder that turns unknown classes into a string and thus never fails """ @@ -39,12 +41,6 @@ class StrJsonEncoder (json.JSONEncoder): except TypeError: return str (obj) -def packageUrl (path): - """ - Create URL for package data stored into WARC - """ - return 'urn:' + __package__ + ':' + urllib.parse.quote (path) - async def getFormattedViewportMetrics (tab): layoutMetrics = await tab.Page.getLayoutMetrics () # XXX: I’m not entirely sure which one we should use here -- cgit v1.2.3