diff options
| -rw-r--r-- | contrib/celerycrocoite.py | 8 | ||||
| -rw-r--r-- | crocoite/cli.py | 6 | ||||
| -rw-r--r-- | crocoite/defaults.py | 27 | ||||
| -rw-r--r-- | crocoite/warc.py | 6 |
4 files changed, 38 insertions, 9 deletions
diff --git a/contrib/celerycrocoite.py b/contrib/celerycrocoite.py index 1923629..6a6ac1c 100644 --- a/contrib/celerycrocoite.py +++ b/contrib/celerycrocoite.py @@ -28,7 +28,7 @@ from sopel.tools import Identifier, SopelMemory import celery from urllib.parse import urlsplit -from crocoite import behavior, cli +from crocoite import behavior, cli, defaults def prettyTimeDelta (seconds): """ @@ -62,7 +62,7 @@ def isValidUrl (s): @nickname_commands ('ao', 'archiveonly') @require_chanmsg () -@require_privilege (VOICE) +#@require_privilege (VOICE) @thread (True) @example ('ao http://example.com') def archive (bot, trigger): @@ -87,8 +87,8 @@ def archive (bot, trigger): 'output': None, 'enabledBehaviorNames': list (behavior.availableNames-blacklistedBehavior), 'browser': None, - 'logBuffer': 1000, - 'maxBodySize': 10*1024*1024, + 'logBuffer': defaults.logBuffer, + 'maxBodySize': defaults.maxBodySize, 'idleTimeout': 10, 'timeout': 1*60*60, # 1 hour } diff --git a/crocoite/cli.py b/crocoite/cli.py index 86fee13..2cbbfa8 100644 --- a/crocoite/cli.py +++ b/crocoite/cli.py @@ -31,7 +31,7 @@ from urllib.parse import urlsplit from celery import Celery from celery.utils.log import get_task_logger -from . import behavior +from . import behavior, defaults from .warc import WarcLoader, SerializingWARCWriter from .browser import ChromeService, NullService from .util import packageUrl, getFormattedViewportMetrics @@ -137,8 +137,8 @@ def main (): parser.add_argument('--distributed', help='Use celery worker', action='store_true') parser.add_argument('--timeout', default=10, type=int, help='Maximum time for archival', metavar='SEC') parser.add_argument('--idle-timeout', default=2, type=int, help='Maximum idle seconds (i.e. no requests)', dest='idleTimeout', metavar='SEC') - parser.add_argument('--log-buffer', default=1000, type=int, dest='logBuffer', metavar='LINES') - parser.add_argument('--max-body-size', default=10*1024*1024, type=int, dest='maxBodySize', help='Max body size', metavar='BYTES') + parser.add_argument('--log-buffer', default=defaults.logBuffer, type=int, dest='logBuffer', metavar='LINES') + parser.add_argument('--max-body-size', default=defaults.maxBodySize, type=int, dest='maxBodySize', help='Max body size', metavar='BYTES') #parser.add_argument('--keep-tab', action='store_true', default=False, dest='keepTab', help='Keep tab open') parser.add_argument('--behavior', help='Comma-separated list of enabled behavior scripts', dest='enabledBehaviorNames', diff --git a/crocoite/defaults.py b/crocoite/defaults.py new file mode 100644 index 0000000..d55312d --- /dev/null +++ b/crocoite/defaults.py @@ -0,0 +1,27 @@ +# Copyright (c) 2017 crocoite contributors +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +""" +Defaults settings +""" + +maxBodySize = 50*1024*1024 +logBuffer = 1000 + diff --git a/crocoite/warc.py b/crocoite/warc.py index 540f673..e04bee4 100644 --- a/crocoite/warc.py +++ b/crocoite/warc.py @@ -40,6 +40,7 @@ from warcio.warcwriter import WARCWriter from .browser import AccountingSiteLoader from .util import packageUrl +from . import defaults class SerializingWARCWriter (WARCWriter): """ @@ -103,8 +104,9 @@ class WARCLogHandler (BufferingHandler): class WarcLoader (AccountingSiteLoader): def __init__ (self, browser, url, writer, - logger=logging.getLogger(__name__), logBuffer=1000, - maxBodySize=10*1024*1024): + logger=logging.getLogger(__name__), + logBuffer=defaults.logBuffer, + maxBodySize=defaults.maxBodySize): super ().__init__ (browser, url, logger) self.writer = writer self.maxBodySize = maxBodySize |