Spelling Correction using Spylls#

This tutorial is available as an IPython notebook at Malaya/example/spelling-correction-spylls.

Dependencies#

Spylls is Hunspell ported to Python.

Before you able to use this spelling correction, you need to install https://github.com/zverok/spylls,

pip3 install Spylls

[1]:

import malaya

/home/husein/dev/malaya/malaya/tokenizer.py:214: FutureWarning: Possible nested set at position 3397
  self.tok = re.compile(r'({})'.format('|'.join(pipeline)))
/home/husein/dev/malaya/malaya/tokenizer.py:214: FutureWarning: Possible nested set at position 3927
  self.tok = re.compile(r'({})'.format('|'.join(pipeline)))

[2]:

# some text examples copied from Twitter

string1 = 'krajaan patut bagi pencen awal skt kpd warga emas supaya emosi'
string2 = 'Husein ska mkn aym dkat kampng Jawa'
string3 = 'Melayu malas ni narration dia sama je macam men are trash. True to some, false to some.'
string4 = 'Tapi tak pikir ke bahaya perpetuate myths camtu. Nanti kalau ada hiring discrimination despite your good qualifications because of your race tau pulak marah. Your kids will be victims of that too.'
string5 = 'DrM cerita Melayu malas semenjak saya kat University (early 1980s) and now as i am edging towards retirement in 4-5 years time after a career of being an Engineer, Project Manager, General Manager'
string6 = 'blh bntg dlm kls nlp sy, nnti intch'
string7 = 'mulakn slh org boleh ,bila geng tuh kena slhkn jgk xboleh trima .. pelik'

Load Spylls model#

def load(model: str = 'libreoffice-pejam', **kwargs):
    """
    Load a spylls Spell Corrector for Malay.

    Parameters
    ----------
    model : str, optional (default='libreoffice-pejam')
        Model spelling correction supported. Allowed values:

        * ``'libreoffice-pejam'`` - from LibreOffice pEJAm, https://extensions.libreoffice.org/en/extensions/show/3868

    Returns
    -------
    result: malaya.spelling_correction.spylls.Spylls class
    """

[5]:

model = malaya.spelling_correction.spylls.load()

List possible generated pool of words#

def edit_candidates(self, word):
    """
    Generate candidates given a word.

    Parameters
    ----------
    word: str

    Returns
    -------
    result: List[str]
    """

[6]:

model.edit_candidates('mhthir')

[6]:

['Mahathir']

[7]:

model.edit_candidates('smbng')

[7]:

['sbng', 'smbang', 'jmbng', 'cmbng']

To correct a word#

def correct(self, word: str, **kwargs):
    """
    Most probable spelling correction for word.

    Parameters
    ----------
    word: str

    Returns
    -------
    result: str
    """

[8]:

model.correct('suke')

[8]:

'suka'

[9]:

model.correct('kpd')

[9]:

'kpd'

[10]:

model.correct('krajaan')

[10]:

'kerajaan'

To correct a sentence#

def correct_text(self, text: str):
    """
    Correct all the words within a text, returning the corrected text.

    Parameters
    ----------
    text: str

    Returns
    -------
    result: str
    """

[11]:

model.correct_text(string1)

[11]:

'kerajaan patut bagi pencen awal tks kpd warga emas supaya emosi'

[12]:

tokenizer = malaya.tokenizer.Tokenizer()

[13]:

string2

[13]:

'Husein ska mkn aym dkat kampng Jawa'

[14]:

tokenized = tokenizer.tokenize(string2)
model.correct_text(' '.join(tokenized))

[14]:

'Hussein ska mkn aum tkad kampang Jawa'

[15]:

tokenized = tokenizer.tokenize(string3)
model.correct_text(' '.join(tokenized))

[15]:

'Melayu malas ni narration dia sama je macam men are trash . True to some , false to some .'

[16]:

tokenized = tokenizer.tokenize(string5)
model.correct_text(' '.join(tokenized))

[16]:

'DrM cerita Melayu malas semenjak saya kat University ( early 1980s ) and now as i am edging towards retirement ni 4 - 5 years time after a career of being na Engineer , Project Manager , General Manager'

[17]:

tokenized = tokenizer.tokenize(string6)
model.correct_text(' '.join(tokenized))

[17]:

'blh bnth dlm slk nap st , nnti intiha'

[18]:

tokenized = tokenizer.tokenize(string7)
model.correct_text(' '.join(tokenized))

[18]:

'mulakan slh org boleh , bila geng tuh kena knsl jgk boleh trima . . pelik'