topicModelingTickets/preprocessing.py

# -*- coding: utf-8 -*-

from datetime import datetime
print(datetime.now())
from datetime import datetime

import time
import logging
from stop_words import get_stop_words

#import words as words
from nltk.corpus import stopwords as nltk_stopwords
from collections import Counter
import csv
import re
import xml.etree.ElementTree as ET
import spacy
import textacy
from scipy import *
import sys
csv.field_size_limit(sys.maxsize)
import pickle
import configparser as ConfigParser
from miscellaneous import *







import time




from datetime import datetime
import logging
from nltk.corpus import stopwords
import csv
import functools
import re
import xml.etree.ElementTree as ET
import spacy
import textacy
from scipy import *
import sys
csv.field_size_limit(sys.maxsize)


import time

import logging
from nltk.corpus import stopwords
import csv
import functools
import re
import xml.etree.ElementTree as ET
import spacy
import textacy
from scipy import *
import sys
csv.field_size_limit(sys.maxsize)



import pickle




# load config
config_ini = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/config.ini"

config = ConfigParser.ConfigParser()
with open(config_ini) as f:
    config.read_file(f)




REGEX_SPECIALCHAR = r'[`\-=~!#@,.$%^&*()_+\[\]{};\'\\:"|</>?]'
REGEX_TOPLVL = r'\.[a-z]{2,3}(\.[a-z]{2,3})?'


THESAURUS = {}
WORDS = {}
LEMMAS = {}
NOUNS = []
VORNAMEN= []
de_stop_words=[]

#############  filter tokens

def keepPOS(pos_list):
    return lambda tok: tok.pos_ in pos_list


def keepNouns(noun_list=NOUNS):
    return lambda tok: tok.lower_ in noun_list


def removePOS(pos_list):
    return lambda tok: tok.pos_ not in pos_list


def removeWords(words, keep=None):
    if hasattr(keep, '__iter__'):
        for k in keep:
            try:
                words.remove(k)
            except ValueError:
                pass

    return lambda tok: tok.lower_ not in words


def keepENT(ent_list):
    return lambda tok: tok.ent_type_ in ent_list


def removeENT(ent_list):
    return lambda tok: tok.ent_type_ not in ent_list


def remove_words_containing_Numbers():
    return lambda tok: not bool(re.search('\d', tok.lower_))


def remove_words_containing_topLVL():
    return lambda tok: not bool(re.search(REGEX_TOPLVL, tok.lower_))


def remove_words_containing_specialCharacters():
    return lambda tok: not bool(re.search(REGEX_SPECIALCHAR, tok.lower_))


def remove_long_words():
    return lambda tok: not len(tok.lower_) < 2


def remove_short_words():
    return lambda tok: not len(tok.lower_) > 35


def remove_first_names():
    return lambda tok: tok.lower_ not in [name.lower() for name in VORNAMEN]


############# strings

def remove_addresses(string):
    pass  # todo

def lemmatizeWord(word,lemma_dict=LEMMAS,n=3):
    for i in range(n):
        try:
            word = lemma_dict[word.lower()] if word.lower() in lemma_dict.keys() else word.lower()
        except:
            print(word)
    return word

def getFirstSynonym(word, thesaurus=THESAURUS):
    if not isinstance(word, str):
        return str(word)

    word = word.lower()

    if word in thesaurus.keys():
        return thesaurus[word]
    else:
        return str(word)



##########################    Spellchecking        ##########################################
# http://norvig.com/spell-correct.html
# http://wortschatz.uni-leipzig.de/en/download

import re
from collections import Counter


def words(text): return re.findall(r'\w+', text.lower())

def P(word, N=sum(WORDS.values())):
    "Probability of `word`."
    return WORDS[word] / N


def correction(word):
    "Most probable spelling correction for word."
    return max(candidates(word), key=P)


def candidates(word):
    "Generate possible spelling corrections for word."
    return (known([word]) or known(edits1(word)) or known(edits2(word)) or [word])


def known(words):
    "The subset of `words` that appear in the dictionary of WORDS."
    return set(w for w in words if w in WORDS)


def edits1(word):
    "All edits that are one edit away from `word`."
    letters = 'abcdefghijklmnopqrstuvwxyz'
    splits = [(word[:i], word[i:]) for i in range(len(word) + 1)]
    deletes = [L + R[1:] for L, R in splits if R]
    transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R) > 1]
    replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
    inserts = [L + c + R for L, R in splits for c in letters]
    return set(deletes + transposes + replaces + inserts)


def edits2(word):
    "All edits that are two edits away from `word`."
    return (e2 for e1 in edits1(word) for e2 in edits1(e1))


def autocorrectWord(word):
    try:
        return correction(word)
    except:
        return word


############# stringcleaning

def stringcleaning(stringstream):


    for string in stringstream:
        string = string.lower()

        # fixUnicode
        string = textacy.preprocess.fix_bad_unicode(string.lower(), normalization=u'NFC')

        # remove_words_containing_topLVL
        string = " ".join([w.lower() for w in string.split() if not re.search(REGEX_TOPLVL, w)])

        # replaceRockDots
        string = re.sub(r'[ß]', "ss", string)
        string = re.sub(r'[ö]', "oe", string)
        string = re.sub(r'[ü]', "ue", string)
        string = re.sub(r'[ä]', "ae", string)

        # seperate_words_on_regex:
        string = " ".join(re.compile(REGEX_SPECIALCHAR).split(string))

        # cut_after
        word = "gruss"
        string = string.rpartition(word)[0] if word in string else string

        # lemmatize
        string = " ".join([lemmatizeWord(word) for word in string.split()])

        # synonyme normalisieren    #idee vor oder nach lemmatize?
        string = " ".join([getFirstSynonym(word) for word in string.split()])

        # autocorrect
        string = " ".join([autocorrectWord(word) for word in string.split()])

        yield string

def filterTokens(tokens, funclist):
    # in:tokenlist, funclist
    # out: tokenlist
    for f in funclist:
        tokens = list(filter(f, tokens))

    return tokens





def corpus2Text(corpus):
    for doc in corpus:
        yield doc.text

def corpus2Meta(corpus):
    for doc in corpus:
        yield doc.metadata

def processContentstream(textstream, parser, token_filterlist=None):
    """
    :param textstream: string-gen
    :param funclist: [func]
    :param parser: spacy-parser
    :return: string-gen
    """

    # pre_parse
    textstream = stringcleaning(textstream)

    pipe = parser.pipe(textstream)

    tokens = []
    for doc in pipe:

        tokens = [tok for tok in doc]

        # in_parse
        if token_filterlist is not None:
            tokens = filterTokens(tokens, token_filterlist)

        yield " ".join([tok.lower_ for tok in tokens])
        # yield " ".join(list(set([tok.lower_ for tok in tokens])))

def processDictstream(dictstream, funcdict, parser):
    """

    :param dictstream: dict-gen
    :param funcdict:
                    clean_in_meta = {
                        "Solution":funclist,
                        ...
                    }

    :param parser: spacy-parser
    :return: dict-gen
    """
    for dic in dictstream:
        result = {}
        for key, value in dic.items():

            if key in funcdict:

                doc = parser(value)
                tokens = [tok for tok in doc]
                funclist = funcdict[key]

                tokens = filterTokens(tokens, funclist)

                result[key] = " ".join([tok.lower_ for tok in tokens])


            else:
                result[key] = value
        yield result


##################################################################################################

# ssh madonna "nohup /usr/bin/python3 -u /home/jannis.grundmann/PycharmProjects/topicModelingTickets/preprocessing.py &> /home/jannis.grundmann/PycharmProjects/topicModelingTickets/printout_preprocessing.log &"


path2thesaurus_dict = config.get("thesaurus","pickle_file")

path2wordsdict = config.get("spellchecking", "pickle_file")

path2lemmadict = config.get("lemmatization","pickle_file")

path2nouns_list = config.get("nouns","pickle_file")

path2firstnameslist = config.get("firstnames","pickle_file")

path2stopwordlist = config.get("de_stopwords","pickle_file")



corpus_de_path = config.get("de_corpus", "path")
raw_de_name = config.get("de_corpus", "raw")
pre_de_name = config.get("de_corpus", "pre")



corpus_en_path = config.get("en_corpus", "path")
raw_en_name = config.get("en_corpus", "raw")
pre_en_name = config.get("en_corpus", "pre")


custom_words = ["geehrt", "dame", "herr", "hilfe", "problem", "lauten", "bedanken", "voraus",
                "hallo", "gerne", "freundlich", "fragen", "fehler", "bitten", "ehre", "lieb", "helfen",
                "versuchen", "unbestimmt", "woche", "tadelos", "klappen", "mittlerweile", "bekommen",
                "erreichbar", "gruss", "auffahren", "vorgang", "hinweis", "institut", "universitaet",
                "name", "gruss", "id", "erfolg", "mail","folge",
                "nummer", "team", "fakultaet", "email", "absender", "tu", "versenden", "vorname", "message",
                "service", "strasse", "prozess", "portal", "raum", "personal", "moeglichkeit", "fremd", "wende",
                "rueckfrage", "stehen", "verfuegung",
                "funktionieren", "kollege", "pruefen", "hoffen"
                ]


filter_tokens = [
    # removeENT(["PERSON"]),
    # idee addressen enfernen #bisher mit cut_after("gruss") --> postal.parser

    keepNouns(),

    remove_words_containing_Numbers(),

    removePOS(["PUNCT", "SPACE", "NUM"]),

    #removeWords(de_stop_words + custom_words),
    removeWords(de_stop_words),

    remove_long_words(),
    remove_short_words(),
    remove_first_names()


]



clean_in_meta = {
    "Solution": [removePOS(["SPACE"])],
    "Subject": [removePOS(["SPACE", "PUNCT"])],
    "categoryName": [removePOS(["SPACE", "PUNCT"])]
}









def main():
    start = time.time()
    printlog("Preprocessing: {0}".format(datetime.now()))


    THESAURUS = load_obj(path2thesaurus_dict)
    WORDS = load_obj(path2wordsdict)
    LEMMAS = load_obj(path2lemmadict)
    DE_STOP_WORDS = load_obj(path2stopwordlist)
    NOUNS = load_obj(path2nouns_list)
    VORNAMEN = load_obj(path2firstnameslist)



    #load raw corpus and create new one
    raw_de_corpus, DE_PARSER = load_corpus(corpus_name=raw_de_name, corpus_path=corpus_de_path)
    raw_en_corpus, EN_PARSER = load_corpus(corpus_name=raw_en_name, corpus_path=corpus_en_path)

    de_corpus = textacy.Corpus(DE_PARSER)
    en_corpus = textacy.Corpus(EN_PARSER)



    ## process and add files to textacy-corpi,
    printlog("Preprocess and add texts to textacy-corpi")
    de_corpus.add_texts(
        processContentstream(corpus2Text(raw_de_corpus), token_filterlist=filter_tokens, parser=DE_PARSER),
        processDictstream(corpus2Meta(raw_de_corpus), clean_in_meta,parser=raw_de_corpus.lang)
    )
    en_corpus.add_texts(
        processContentstream(corpus2Text(raw_en_corpus), token_filterlist=filter_tokens, parser=EN_PARSER),
        processDictstream(corpus2Meta(raw_en_corpus), clean_in_meta,parser=raw_en_corpus.lang)
    )


    # leere docs aus corpi kicken
    de_corpus.remove(lambda doc: len(doc) == 0)
    en_corpus.remove(lambda doc: len(doc) == 0)


    for i in range(20):
        printRandomDoc(de_corpus)
        #printRandomDoc(en_corpus)



    #save corpi
    save_corpus(corpus=de_corpus, corpus_path=corpus_de_path, corpus_name=pre_de_name)
    save_corpus(corpus=en_corpus, corpus_path=corpus_en_path, corpus_name=pre_en_name)



    end = time.time()
    printlog("Time Elapsed Preprocessing:{0} min".format((end - start) / 60))

if __name__ == "__main__":
    main()







"""
pipe=[

    ##String

    fixUnicode(),
    replaceHardS(),
    resolveAbbrivations(),

    remove_words_containing_topLVL(),

    replaceSpecialChars(" "), (mit Leerzeichen erstzen, dadruch werden Terme wie 8203;verfügung getrennt

    remove_words_containing_Numbers(),



    ##spacyParse

    removeENT("PERSON"),
    keepPOS(["NOUN"]),

    #ODER

    lemmatize(),
    removeWords(de_stop_words + config.get("preprocessing","custom_words").split(",")),


    # evtl.
    spellCorrection(),
    keepUniqeTokens(),

]




"""

"""
filter_tokens=[
#removeENT(["PERSON"]),
#idee addressen enfernen #bisher mit cut_after("gruss") --> postal.parser
#idee rechtschreibkorrektur --> PyEnchant
#idee thesaurus --> WordNet, eigener

remove_words_containing_Numbers(),

removePOS(["PUNCT","SPACE","NUM"]),

removeWords(de_stop_words+custom_words),

remove_long_words(),
remove_short_words(),
remove_first_names(),

keepPOS(["NOUN"]),

]
"""