From 38c9ed5b042ae488ee12287bf8c19457189889aa Mon Sep 17 00:00:00 2001 From: Lars-Dominik Braun Date: Fri, 8 Nov 2019 16:06:37 +0100 Subject: Add OpenSubtitles corpus See issue #5. --- lulua/text.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'lulua/text.py') diff --git a/lulua/text.py b/lulua/text.py index 182c717..382877b 100644 --- a/lulua/text.py +++ b/lulua/text.py @@ -185,6 +185,23 @@ def sourceTEI2 (item): except Exception: logging.error (f'invalid xml document {item}') +def sourceOpenSubtitles (item): + """ + XML-based format used by the (raw!) OpenSubtitles dump found here: + http://opus.nlpl.eu/OpenSubtitles-v2018.php + """ + with open (item.rstrip (), 'rb') as fd: + try: + out = [] + doc = xml.dom.minidom.parse (fd) + for s in doc.getElementsByTagName ('s'): + # strip newlines, which are mostly unintentional due to + # pretty-printed xml structure + out.append (getText (s.childNodes).strip ()) + yield '\n'.join (out) + except Exception as e: + logging.error (f'invalid xml document {item} {e}') + sources = dict( aljazeera=partial(sourceHtml, f['aljazeera']), bbcarabic=partial(sourceHtml, f['bbcarabic']), @@ -192,6 +209,7 @@ sources = dict( json=sourceJson, epub=sourceEpub, tei2=sourceTEI2, + opensubtitles=sourceOpenSubtitles, ) charMap = { -- cgit v1.2.3