diff options
author | Lars-Dominik Braun <lars@6xq.net> | 2018-12-18 12:34:25 +0100 |
---|---|---|
committer | Lars-Dominik Braun <lars@6xq.net> | 2018-12-21 20:28:51 +0100 |
commit | 5e444dd6511d97308a84ae9c86ebf14547d01f01 (patch) | |
tree | 0852c081163ff3456038fb08ad4e47d0d47a6167 /crocoite/util.py | |
parent | e19635a75cc1ab206be12ecf2b1c9a909baa9c21 (diff) | |
download | crocoite-5e444dd6511d97308a84ae9c86ebf14547d01f01.tar.gz crocoite-5e444dd6511d97308a84ae9c86ebf14547d01f01.tar.bz2 crocoite-5e444dd6511d97308a84ae9c86ebf14547d01f01.zip |
Parse URLs by default
Use library yarl (already pulled in by aiohttp). No URL processed should
be a string.
Diffstat (limited to 'crocoite/util.py')
-rw-r--r-- | crocoite/util.py | 22 |
1 files changed, 15 insertions, 7 deletions
diff --git a/crocoite/util.py b/crocoite/util.py index bd26909..73a1d65 100644 --- a/crocoite/util.py +++ b/crocoite/util.py @@ -22,9 +22,22 @@ Random utility functions """ -import random, sys, platform +import random, sys, platform, os, json +from datetime import datetime import hashlib, pkg_resources -from urllib.parse import urlsplit, urlunsplit + +class StrJsonEncoder (json.JSONEncoder): + """ JSON encoder that turns unknown classes into a string and thus never + fails """ + def default (self, obj): + if isinstance (obj, datetime): + return obj.isoformat () + + # make sure serialization always succeeds + try: + return json.JSONEncoder.default(self, obj) + except TypeError: + return str (obj) def packageUrl (path): """ @@ -38,11 +51,6 @@ async def getFormattedViewportMetrics (tab): return '{}x{}'.format (layoutMetrics['layoutViewport']['clientWidth'], layoutMetrics['layoutViewport']['clientHeight']) -def removeFragment (u): - """ Remove fragment from url (i.e. #hashvalue) """ - s = urlsplit (u) - return urlunsplit ((s.scheme, s.netloc, s.path, s.query, '')) - def getSoftwareInfo (): """ Get software info for inclusion into warcinfo """ return { |