topicModelingTickets/preprocessing.py

# -*- coding: utf-8 -*-

from datetime import datetime
import csv
import sys
from miscellaneous import *
from datetime import datetime
import time
import textacy
from scipy import *

import os

csv.field_size_limit(sys.maxsize)
FILEPATH = os.path.dirname(os.path.realpath(__file__)) + "/"



# load config
config_ini = FILEPATH + "config.ini"

config = ConfigParser.ConfigParser()
with open(config_ini) as f:
    config.read_file(f)




REGEX_SPECIALCHAR = r'[`\-=~%^&*()_+\[\]{};\'\\:"|</>]' #+r',.'
REGEX_TOPLVL = r'\.[a-z]{2,3}(\.[a-z]{2,3})?'


THESAURUS = {}
WORDS= {}
LEMMAS= {}
NOUNS= {}
VORNAMEN= {}
DE_STOP_WORDS= {}
EN_STOP_WORDS= {}



#############  filter tokens


def filterTokens(tokens, funclist):
    # in:tokenlist, funclist
    # out: tokenlist
    for f in funclist:
        tokens = list(filter(f, tokens))

    for tok in tokens:
        if tok.pos_ =="NOUN":
            x=0

    return tokens


def keepPOS(pos_list):
    return lambda tok: tok.pos_ in pos_list


def keepNouns(noun_list=NOUNS):
    #return lambda tok: tok.lower_ in noun_list
    return lambda tok: tok.lower_ in noun_list or tok.pos_ == "NOUN"



def removePOS(pos_list):
    return lambda tok: tok.pos_ not in pos_list


def removeWords(words, keep=None):
    if hasattr(keep, '__iter__'):
        for k in keep:
            try:
                words.remove(k)
            except ValueError:
                pass

    return lambda tok: tok.lower_ not in words


def keepENT(ent_list):
    return lambda tok: tok.ent_type_ in ent_list


def removeENT(ent_list):
    return lambda tok: tok.ent_type_ not in ent_list


def remove_words_containing_Numbers():
    return lambda tok: not bool(re.search('\d', tok.lower_))


def remove_words_containing_topLVL():
    return lambda tok: not bool(re.search(REGEX_TOPLVL, tok.lower_))


def remove_words_containing_specialCharacters():
    return lambda tok: not bool(re.search(REGEX_SPECIALCHAR, tok.lower_))


def remove_long_words():
    return lambda tok: not len(tok.lower_) < 2


def remove_short_words():
    return lambda tok: not len(tok.lower_) > 35


def remove_first_names():
    return lambda tok: tok.lower_ not in [name.lower() for name in VORNAMEN]


############# strings

def remove_addresses(string):
    pass  # todo remove_addresses  idee postal.parser und zu metadaten hinzufügen

def lemmatizeWord(word,lemma_dict=LEMMAS,n=3):
    for i in range(n):
        try:
            word = lemma_dict[word.lower()] if word.lower() in lemma_dict.keys() else word.lower()
        except:
            print(word)
    return word

def getFirstSynonym(word, thesaurus=THESAURUS):
    if not isinstance(word, str):
        return str(word)

    word = word.lower()

    if word in thesaurus.keys():
        return thesaurus[word]
    else:
        return str(word)



##########################    Spellchecking        ##########################################
# http://norvig.com/spell-correct.html
# http://wortschatz.uni-leipzig.de/en/download

import re


def words(text): return re.findall(r'\w+', text.lower())

def P(word, N=sum(WORDS.values())):
    "Probability of `word`."
    return WORDS[word] / N


def correction(word):
    "Most probable spelling correction for word."
    return max(candidates(word), key=P)


def candidates(word):
    "Generate possible spelling corrections for word."
    return (known([word]) or known(edits1(word)) or known(edits2(word)) or [word])


def known(words):
    "The subset of `words` that appear in the dictionary of WORDS."
    return set(w for w in words if w in WORDS)


def edits1(word):
    "All edits that are one edit away from `word`."
    letters = 'abcdefghijklmnopqrstuvwxyz'
    splits = [(word[:i], word[i:]) for i in range(len(word) + 1)]
    deletes = [L + R[1:] for L, R in splits if R]
    transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R) > 1]
    replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
    inserts = [L + c + R for L, R in splits for c in letters]
    return set(deletes + transposes + replaces + inserts)


def edits2(word):
    "All edits that are two edits away from `word`."
    return (e2 for e1 in edits1(word) for e2 in edits1(e1))


def autocorrectWord(word):
    try:
        return correction(word)
    except:
        return word


############# stringcleaning


def processContentstream(textstream, parser, token_filterlist=None):

    #pre parse
    textstream = preparse(textstream)

    pipe = parser.pipe(textstream)

    for doc in pipe:

        tokens = [tok for tok in doc]

        # in parse
        if token_filterlist is not None:
            tokens = filterTokens(tokens, token_filterlist)

        # post parse
        #todo STELLSCHRAUBE tokens = [postparse(tok) for tok in tokens]   #todo: informationsverlust von pos,tag etc.!
        tokens =  [tok.lower_ for tok in tokens]
        yield " ".join(tokens)

def preparse(stringstream):

    for string in stringstream:

        # cut_after
        words = ["gruss", "grusse","gruesse","gruessen","grusses"]

        for gr in words:
            if gr in string:
                string = string.rpartition(gr)[0]
                break

        yield string

def postparse(toktext):
    """
    :param toktext: spacy.token
    :return: string
    """
    toktext = toktext.lower_

    # remove_words_containing_topLVL
    toktext = toktext if not re.search(REGEX_TOPLVL, toktext) else ""

    # lemmatize
    toktext = lemmatizeWord(toktext)

    # synonyme normalisieren
    toktext = getFirstSynonym(toktext)

    # autocorrect
    toktext = autocorrectWord(toktext)

    return toktext


def processDictstream(dictstream, funcdict, parser):
    """

    :param dictstream: dict-gen
    :param funcdict:
                    clean_in_meta = {
                        "Solution":funclist,
                        ...
                    }

    :param parser: spacy-parser
    :return: dict-gen
    """
    for dic in dictstream:
        result = {}
        for key, value in dic.items():

            if key in funcdict:

                doc = parser(value)
                tokens = [tok for tok in doc]
                funclist = funcdict[key]

                tokens = filterTokens(tokens, funclist)

                result[key] = " ".join([tok.lower_ for tok in tokens])


            else:
                result[key] = value
        yield result


##################################################################################################


ressources_path = FILEPATH +  "ressources/"

path2thesaurus_dict = ressources_path + config.get("thesaurus","pickle_file")
path2wordsdict = ressources_path + config.get("spellchecking", "pickle_file")
path2lemmadict = ressources_path + config.get("lemmatization","pickle_file")
path2nouns_list = ressources_path + config.get("nouns","pickle_file")
path2firstnameslist = ressources_path + config.get("firstnames","pickle_file")


path2DEstopwordlist = ressources_path + config.get("de_stopwords", "pickle_file")

path2ENstopwordlist = ressources_path + config.get("en_stopwords", "pickle_file")

custom_words = get_list_from_config("preprocessing", "custom_words")



corpus_de_path = FILEPATH + config.get("de_corpus", "path")
de_plainpath = FILEPATH + config.get("de_corpus", "path") + "pre_labled_lines.txt"

corpus_en_path = FILEPATH + config.get("en_corpus", "path")



def preprocessCorpus(corpus_path, filter_tokens, clean_in_meta, lang="de", printrandom=10):

    logprint("Preprocess {0}_corpus at {1}".format(lang, datetime.now()))

    cleanCorpus_name = lang + "_clean_ticket"
    preCorpus_name = lang + "_pre_ticket"

    #load raw corpus and create new one
    logprint("Load {0}_raw".format(lang))
    clean_corpus, parser = load_corpus(corpus_name=cleanCorpus_name, corpus_path=corpus_path)

    corpus = textacy.Corpus(parser)


    ## process and add files to textacy-corpi,
    corpus.add_texts(
        processContentstream(corpus2Text(clean_corpus), token_filterlist=filter_tokens, parser=parser),
        processDictstream(corpus2Meta(clean_corpus), clean_in_meta,parser=parser)
    )


    # leere docs aus corpi kicken
    corpus.remove(lambda doc: len(doc) == 0)


    for i in range(printrandom):
        printRandomDoc(corpus)


    #save corpus
    save_corpus(corpus=corpus, corpus_path=corpus_path, corpus_name=preCorpus_name)


    #save corpus as labled, plain text
    savelabledCorpiLines(corpus, de_plainpath)


    return corpus


def main():
    start = time.time()


    THESAURUS = load_obj(path2thesaurus_dict)
    WORDS = load_obj(path2wordsdict)
    LEMMAS = load_obj(path2lemmadict)
    DE_STOP_WORDS = load_obj(path2DEstopwordlist)
    EN_STOP_WORDS = load_obj(path2ENstopwordlist)
    NOUNS = load_obj(path2nouns_list)
    VORNAMEN = load_obj(path2firstnameslist)


    filter_tokens = [

        keepNouns(NOUNS),

        removeWords(DE_STOP_WORDS + custom_words + VORNAMEN),

        removePOS(["PUNCT", "SPACE", "NUM"]),

        #todo STELLSCHRAUBE remove_words_containing_Numbers(),
        #todo STELLSCHRAUBE remove_long_words(),
        #todo STELLSCHRAUBE remove_short_words()

    ]


    clean_in_meta = {
        "Solution": [removePOS(["SPACE"])],
        "Subject": [removePOS(["SPACE", "PUNCT"])],
        "categoryName": [removePOS(["SPACE", "PUNCT"])]
    }


    corpus = preprocessCorpus(corpus_de_path,  filter_tokens, clean_in_meta, "de",printrandom=5)



    end = time.time()
    logprint("Time Elapsed Preprocessing:{0} min".format((end - start) / 60))

if __name__ == "__main__":
    main()







"""
pipe=[

    ##String

    fixUnicode(),
    replaceHardS(),
    resolveAbbrivations(),

    remove_words_containing_topLVL(),

    replaceSpecialChars(" "), (mit Leerzeichen erstzen, dadruch werden Terme wie 8203;verfügung getrennt

    remove_words_containing_Numbers(),



    ##spacyParse

    removeENT("PERSON"),
    keepPOS(["NOUN"]),

    #ODER

    lemmatize(),
    removeWords(de_stop_words + config.get("preprocessing","custom_words").split(",")),


    # evtl.
    spellCorrection(),
    keepUniqeTokens(),

]




"""

"""
filter_tokens=[
#removeENT(["PERSON"]),
#idee addressen enfernen #bisher mit cut_after("gruss") --> postal.parser
#idee rechtschreibkorrektur --> PyEnchant
#idee thesaurus --> WordNet, eigener

remove_words_containing_Numbers(),

removePOS(["PUNCT","SPACE","NUM"]),

removeWords(de_stop_words+custom_words),

remove_long_words(),
remove_short_words(),
remove_first_names(),

keepPOS(["NOUN"]),

]
"""