topicModelingTickets/init.py

# -*- coding: utf-8 -*-

from datetime import datetime

import time
import logging
from nltk.corpus import stopwords as nltk_stopwords
from collections import Counter
import csv
import re
import xml.etree.ElementTree as ET
import spacy
import textacy
from scipy import *
import sys
csv.field_size_limit(sys.maxsize)
import pickle


# todo configuration file ?
"""
config_ini = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/config.ini"

config = ConfigParser.ConfigParser()
with open(config_ini) as f:
    config.read_file(f)
"""



# config logging
logfile = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/topicModelTickets.log"
logging.basicConfig(filename=logfile, level=logging.INFO)




DE_PARSER = spacy.load("de")
EN_PARSER = spacy.load("en")



def replaceRockDots():
    return lambda string: re.sub(r'[ß]', "ss",
                                 (re.sub(r'[ö]', "oe", (re.sub(r'[ü]', "ue", (re.sub(r'[ä]', "ae", string.lower())))))))

def printlog(string, level="INFO"):
    """log and prints"""
    print(string)
    if level == "INFO":
        logging.info(string)
    elif level == "DEBUG":
        logging.debug(string)
    elif level == "WARNING":
        logging.warning(string)




def save_obj(obj, path):
    with open(path + '.pkl', 'wb') as f:
        pickle.dump(obj, f, pickle.HIGHEST_PROTOCOL)

def load_obj(path ):
    with open(path + '.pkl', 'rb') as f:
        return pickle.load(f)

def create_lemma_dict(lemmalist):

    lemma_dict = {}

    for line in lemmalist:
        lem_word_pair = line.split()

        lemma = lem_word_pair[0].strip().lower()

        word = lem_word_pair[1].strip().lower()

        lemma_dict[word] = lemma

    return lemma_dict


def build_thesaurus(path2lexicalentries, path2synsets):
    lextree = ET.parse(path2lexicalentries, ET.XMLParser(encoding="utf-8"))
    syntree = ET.parse(path2synsets, ET.XMLParser(encoding="utf-8"))

    lexroot = lextree.getroot()
    synroot = syntree.getroot()

    thesaurus = []

    for r in synroot:
        for element in r:

            if element.tag == "Synset":
                sysnet = []
                attrib = element.attrib
                id = attrib["id"]

                for ro in lexroot:
                    for elem in ro:
                        if elem.tag == "LexicalEntry":
                            subs_dicts = [subentry.attrib for subentry in elem]
                            # <class 'list'>: [{'partOfSpeech': 'n', 'writtenForm': 'Kernspaltung'}, {'synset': 'de-1-n', 'id': 'w1_1-n'}]

                            dic = {k: v for x in subs_dicts for k, v in x.items()}  # to one dict
                            if "synset" in dic.keys():
                                if dic["synset"] == id:
                                    string = (dic["writtenForm"])

                                    # replaceRockDots
                                    string = re.sub(r'[ß]', "ss", string)
                                    string = re.sub(r'[ö]', "oe", string)
                                    string = re.sub(r'[ü]', "ue", string)
                                    string = re.sub(r'[ä]', "ae", string)

                                    # alle punkte raus
                                    string = re.sub(r'[.]', "", string)

                                    # alles in klammern raus
                                    string = re.sub(r"\((.*)\)", " ", string)

                                    # längeres leerzeichen normalisieren
                                    string = textacy.preprocess.normalize_whitespace(string)

                                    sysnet.append(string.lower().strip())

                                    # nach anzhal der wörter in den strings sortieren
                                    sysnet.sort(key=lambda x: len(x.split()))
                if len(sysnet) != 0:
                    # todo warum sind manche leer?
                    thesaurus.append(sysnet)
    return thesaurus

    #todo thesaurus in dictionary


def create_stopwordlist():

    de_stop_words1 = list(map(replaceRockDots(),
                              list(
                                  map(textacy.preprocess.normalize_whitespace,
                                      textacy.fileio.read_file_lines("/home/jannis.grundmann/PycharmProjects/topicModelingTickets/de_stop_words.txt")
                                      )
                                  )
                              )
                          )

    de_stop_words2 = list(map(replaceRockDots(),list(set(nltk_stopwords.words('german')))))

    de_stop_words3 = list(map(replaceRockDots(),list(__import__("spacy." + DE_PARSER.lang, globals(), locals(), ['object']).STOP_WORDS)))

    de_stop_words4 = list(map(replaceRockDots(),list(textacy.fileio.read_file_lines("stopwords-de.txt"))))

    de_stop_words = list(set(de_stop_words1 + de_stop_words2 + de_stop_words3 + de_stop_words4))

    return de_stop_words

    #todo en_stop_words= set(list(__import__("spacy." + EN_PARSER.lang, globals(), locals(), ['object']).STOP_WORDS)+ list(set(nltk_stopwords.words('english'))))





##########################    Spellchecking        ##########################################
# http://norvig.com/spell-correct.html
# http://wortschatz.uni-leipzig.de/en/download

def words(text): return re.findall(r'\w+', text.lower())


##################################################################################################

# ziel: dictionaries für thesaurus, correctwordliste und lemmas als ladbare .json
# außerdem saubere stoppwortliste und nomenliste



# THESAURUS
lexicalentries = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/lexicalentries.xml"
synsets = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/synsets.xml"

lexicalentries = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/lexicalentries_small.xml"
synsets = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/synsets.xml"



# SPELLCHECKING
path2words = '/home/jannis.grundmann/PycharmProjects/topicModelingTickets/deu_news_2015_1M-sentences.txt'






path2lemmadict = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/lemma_dict.pkl"
path2wordlist = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/words_list.pkl"
path2thesauruslist = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/thesaurus_list.pkl"
path2stopwordlist = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/stopwords_list.pkl"
path2NOUNSlist = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/nouns_list.pkl"
path2firstnameslist = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/firstnames_list.pkl"







def main():
    start = time.time()
    printlog("Init: {0}".format(datetime.now()))




    printlog("create and save lemma_dict")
    LEMMAS = list(
        textacy.fileio.read_file_lines("/home/jannis.grundmann/PycharmProjects/topicModelingTickets/lemmas.txt"))

    lemma_dict = create_lemma_dict(LEMMAS)
    save_obj(lemma_dict, path2lemmadict)






    printlog("Build and save Wordlist for Spellchecking")
    WORDS = Counter(words(open(path2words).read()))
    save_obj(WORDS, path2wordlist)





    printlog("Build and save Thesaurus")
    THESAURUS = build_thesaurus(path2lexicalentries=lexicalentries, path2synsets=synsets)
    print(THESAURUS[0:10])


    save_obj(THESAURUS, path2thesauruslist)





    printlog("Build and save stoppwortliste")
    de_stop_words = create_stopwordlist()
    save_obj(de_stop_words, path2stopwordlist)






    printlog("Build and save nomenliste")
    NOUNS = list(textacy.fileio.read_file_lines(
        "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/nomen2.txt")) + list(
        textacy.fileio.read_file_lines("/home/jannis.grundmann/PycharmProjects/topicModelingTickets/nomen.txt"))
    NOUNS = list(map(textacy.preprocess.normalize_whitespace, NOUNS))
    save_obj(NOUNS, path2NOUNSlist)






    printlog("Build and save fistnameslist")
    VORNAMEN = list(map(textacy.preprocess.normalize_whitespace, textacy.fileio.read_file_lines(
        "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/firstnames.txt")))

    save_obj(VORNAMEN, path2firstnameslist)






    end = time.time()
    printlog("Time Elapsed Preprocessing:{0} min".format((end - start) / 60))



if __name__ == "__main__":
    main()