From 5e444dd6511d97308a84ae9c86ebf14547d01f01 Mon Sep 17 00:00:00 2001
From: Lars-Dominik Braun <lars@6xq.net>
Date: Tue, 18 Dec 2018 12:34:25 +0100
Subject: Parse URLs by default

Use library yarl (already pulled in by aiohttp). No URL processed should
be a string.
---
 crocoite/util.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

(limited to 'crocoite/util.py')

diff --git a/crocoite/util.py b/crocoite/util.py
index bd26909..73a1d65 100644
--- a/crocoite/util.py
+++ b/crocoite/util.py
@@ -22,9 +22,22 @@
 Random utility functions
 """
 
-import random, sys, platform
+import random, sys, platform, os, json
+from datetime import datetime
 import hashlib, pkg_resources
-from urllib.parse import urlsplit, urlunsplit
+
+class StrJsonEncoder (json.JSONEncoder):
+    """ JSON encoder that turns unknown classes into a string and thus never
+    fails """
+    def default (self, obj):
+        if isinstance (obj, datetime):
+            return obj.isoformat ()
+
+        # make sure serialization always succeeds
+        try:
+            return json.JSONEncoder.default(self, obj)
+        except TypeError:
+            return str (obj)
 
 def packageUrl (path):
     """
@@ -38,11 +51,6 @@ async def getFormattedViewportMetrics (tab):
     return '{}x{}'.format (layoutMetrics['layoutViewport']['clientWidth'],
                 layoutMetrics['layoutViewport']['clientHeight'])
 
-def removeFragment (u):
-    """ Remove fragment from url (i.e. #hashvalue) """
-    s = urlsplit (u)
-    return urlunsplit ((s.scheme, s.netloc, s.path, s.query, ''))
-
 def getSoftwareInfo ():
     """ Get software info for inclusion into warcinfo """
     return {
-- 
cgit v1.2.3