summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLars-Dominik Braun <lars@6xq.net>2017-11-22 08:56:25 +0100
committerLars-Dominik Braun <lars@6xq.net>2017-11-22 08:56:25 +0100
commit82d779208d208c225e7b3deefc3aa45bb0aedce8 (patch)
tree1164adb0bfd301b011c172a91e43a0f99cea5a7d
parent8c75a5cc52605348df3718ea332e74c15fc0b358 (diff)
downloadcrocoite-82d779208d208c225e7b3deefc3aa45bb0aedce8.zip
crocoite-82d779208d208c225e7b3deefc3aa45bb0aedce8.tar.gz
crocoite-82d779208d208c225e7b3deefc3aa45bb0aedce8.tar.bz2
Add example fixups for Instagram
-rw-r--r--crocoite/cli.py12
-rw-r--r--crocoite/data/fixups.instagram.js21
-rw-r--r--crocoite/data/scroll.js2
3 files changed, 32 insertions, 3 deletions
diff --git a/crocoite/cli.py b/crocoite/cli.py
index 54a31d9..9a22d24 100644
--- a/crocoite/cli.py
+++ b/crocoite/cli.py
@@ -327,7 +327,7 @@ def main ():
parser.add_argument('--browser', default='http://127.0.0.1:9222', help='DevTools URL')
parser.add_argument('--timeout', default=10, type=int, help='Maximum time for archival')
parser.add_argument('--idle-timeout', default=2, type=int, help='Maximum idle seconds (i.e. no requests)', dest='idleTimeout')
- parser.add_argument('--onload', default=packageData ('scroll.js'), help='')
+ parser.add_argument('--onload', action='append', help='')
parser.add_argument('--log-buffer', default=1000, type=int, dest='logBuffer')
parser.add_argument('--keep-tab', action='store_true', default=False, dest='keepTab', help='Keep tab open')
parser.add_argument('url', help='Website URL')
@@ -338,8 +338,14 @@ def main ():
stopVarname = '__' + __package__ + '_stop__'
# avoid sites messing with our scripts by using a random stop variable name
newStopVarname = randomString ()
- with open (args.onload, 'r') as fd:
- onload = 'var {} = false;\n'.format (newStopVarname) + fd.read ().replace (stopVarname, newStopVarname)
+ onload = ['var {} = false;\n'.format (newStopVarname)]
+ for path in args.onload:
+ if not os.path.exists (path):
+ # search for defaults scripts in package data directory
+ path = packageData (path)
+ with open (path, 'r') as fd:
+ onload.append (fd.read ().replace (stopVarname, newStopVarname))
+ onload = '\n'.join (onload)
stopVarname = newStopVarname
# temporary store for requests
diff --git a/crocoite/data/fixups.instagram.js b/crocoite/data/fixups.instagram.js
new file mode 100644
index 0000000..da7b5ea
--- /dev/null
+++ b/crocoite/data/fixups.instagram.js
@@ -0,0 +1,21 @@
+/* Fixups for instagram: searches for the “show more” button and clicks it
+ */
+(function(){
+function fixup () {
+ var links = document.querySelectorAll ("main a");
+ for (var i = 0; i < links.length; i++) {
+ var href = links[i].getAttribute ("href");
+ if (href.search (/\?max_id=\d+$/) != -1) {
+ var click = new MouseEvent('click', {
+ view: window,
+ bubbles: true,
+ cancelable: true
+ });
+ console.log ('clicking', href);
+ links[i].dispatchEvent (click);
+ break;
+ }
+ }
+}
+window.addEventListener("load", fixup);
+}());
diff --git a/crocoite/data/scroll.js b/crocoite/data/scroll.js
index d16383c..2b4dff1 100644
--- a/crocoite/data/scroll.js
+++ b/crocoite/data/scroll.js
@@ -1,3 +1,5 @@
+/* Continuously scrolls the page
+ */
(function(){
function scroll (event) {
if (__crocoite_stop__) {