Sentence tokenizer

This tutorial is available as an IPython notebook at Malaya/example/tokenizer-sentence.

%%time
import malaya

CPU times: user 6.15 s, sys: 1.22 s, total: 7.36 s
Wall time: 8.81 s

Sentence tokenizer

We considered prefixes, suffixes, starters, acronyms, websites, emails, digits, before digits, titles, time and month to split a sentence into multiple sentences.

class SentenceTokenizer:
    def __init__(self):
        pass

s_tokenizer = malaya.tokenizer.SentenceTokenizer()

Tokenize

def tokenize(self, string, minimum_length=5):
    """
    Tokenize string into multiple strings.

    Parameters
    ----------
    string : str
    minimum_length: int, optional (default=5)
        minimum length to assume a string is a string, default 5 characters.

    Returns
    -------
    result: List[str]
    """

s = """
no. 1 polis bertemu dengan suspek di ladang getah. polis tembak pui pui pui bertubi tubi
"""
s_tokenizer.tokenize(s)

['no.1 polis bertemu dengan suspek di ladang getah.',
 'polis tembak pui pui pui bertubi tubi.']

s = """
email saya di husein.zol01@gmail.com, nanti jom berkopi
"""
s_tokenizer.tokenize(s)

['email saya di husein.zol01@gmail.com, nanti jom berkopi.']

s = """
ke. 2 cerita nya begini. saya berjalan jalan ditepi muara jumpa anak dara.
"""
s_tokenizer.tokenize(s)

['ke.2 cerita nya begini.',
 'saya berjalan jalan ditepi muara jumpa anak dara.']

s = """
ke. 2 cerita nya begini. aku jumpa ybhg. dr. syed tadi, sakai gila
"""
s_tokenizer.tokenize(s)

['ke.2 cerita nya begini.', 'aku jumpa ybhg. dr. syed tadi, sakai gila.']