Add OpenStreetMap label corpus

Extract node labels (name:ar) from OpenStreetMap’s planet dump. Heavily
leans towards a few common words (“street”, obviously), but we should be
fine since the corpus is not that large.

1 files changed, 5 insertions, 0 deletions

diff --git a/lulua/text.py b/lulua/text.py
index 382877b..2d8398d 100644
--- a/lulua/text.py
+++ b/lulua/text.py
@@ -159,6 +159,10 @@ def sourceText (item):
     with LzipFile (item.rstrip ()) as fd:
         yield fd.read ().decode ('utf-8')
 
+def sourceLines (item):
+    """ Read items (i.e. lines) as is """
+    yield item
+
 def sourceJson (item):
     yield json.loads (item)
 
@@ -210,6 +214,7 @@ sources = dict(
     epub=sourceEpub,
     tei2=sourceTEI2,
     opensubtitles=sourceOpenSubtitles,
+    lines=sourceLines,
     )
 
 charMap = {