topicModelingTickets/preprocessing.py

# -*- coding: utf-8 -*-
import csv
import functools
import os.path
import re
import subprocess
import time
import xml.etree.ElementTree as ET

import spacy
import textacy
from scipy import *
from textacy import Vectorizer

csv.field_size_limit(sys.maxsize)



# Load the configuration file
import configparser as ConfigParser
config = ConfigParser.ConfigParser()
with open("config.ini") as f:
    config.read_file(f)



path2xml = config.get("default","path2xml")

PARSER = spacy.load(config.get("default","language"))
corpus = textacy.Corpus(PARSER)

thesauruspath = config.get("default","thesauruspath")
THESAURUS = list(textacy.fileio.read_csv(thesauruspath, delimiter=";"))

stop_words=list(__import__("spacy." + PARSER.lang, globals(), locals(), ['object']).STOP_WORDS) + config.get("preprocessing","custom_words").split(",")


def compose(*functions):
    def compose2(f, g):
        return lambda x: f(g(x))
    return functools.reduce(compose2, functions, lambda x: x)



def generateMainTextfromTicketXML(path2xml, main_textfield='Beschreibung'):
    """
    generates strings from XML
    :param path2xml:
    :param main_textfield:
    :param cleaning_function:
    :yields strings
    """
    tree = ET.parse(path2xml, ET.XMLParser(encoding="utf-8"))
    root = tree.getroot()

    for ticket in root:
        for field in ticket:
            if field.tag == main_textfield:
                    yield field.text

def generateMetadatafromTicketXML(path2xml, leave_out=['Beschreibung']):
    tree = ET.parse(path2xml, ET.XMLParser(encoding="utf-8"))
    root = tree.getroot()

    for ticket in root:
        metadata = {}
        for field in ticket:
            if field.tag not in leave_out:

                    metadata[field.tag] = field.text

        yield metadata

def printRandomDoc(textacyCorpus):
    import random
    print()

    print("len(textacyCorpus) = %i" % len(textacyCorpus))
    randIndex = int((len(textacyCorpus) - 1) * random.random())
    print("Index: {0} ; Text: {1} ; Metadata: {2}".format(randIndex, textacyCorpus[randIndex].text, textacyCorpus[randIndex].metadata))

    print()

def processDictstream(dictstream, funcdict, parser=PARSER):
    for dic in dictstream:
        result = {}
        for key, value in dic.items():
            if key in funcdict:
                result[key] = funcdict[key](parser(value))
            else:
                result[key] = value
        yield result

def processTextstream(textstream, func, parser=PARSER):
   # input str-stream output str-stream
    pipe = parser.pipe(textstream)

    for doc in pipe:
        yield func(doc)




def keepOnlyPOS(pos_list, parser=PARSER):
    return lambda doc : parser(" ".join([tok.text for tok in doc if tok.pos_ in pos_list]))

def removeAllPOS(pos_list, parser=PARSER):
    return lambda doc:  parser(" ".join([tok.text for tok in doc if tok.pos_ not in pos_list]))

def keepOnlyENT(ent_list,parser=PARSER):
    return lambda doc:  parser(" ".join([tok.text for tok in doc if tok.ent_type_ in ent_list]))

def removeAllENT(ent_list, parser=PARSER):
    return lambda doc:  parser(" ".join([tok.text for tok in doc if tok.ent_type_ not in ent_list]))



doc2Set = lambda doc: str(set([tok.text for tok in doc]))
doc2String = lambda doc : doc.text


mentionFinder = re.compile(r"@[a-z0-9_]{1,15}", re.IGNORECASE)
emailFinder = re.compile(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.IGNORECASE)
urlFinder = re.compile(r"^(?:https?:\/\/)?(?:www\.)?[a-zA-Z0-9./]+$", re.IGNORECASE)

def replaceURLs(replace_with="URL",parser=PARSER):
    #return lambda doc: parser(textacy.preprocess.replace_urls(doc.text,replace_with=replace_with))
    return lambda doc: parser(urlFinder.sub(replace_with,doc.text))

def replaceEmails(replace_with="EMAIL",parser=PARSER):
    #return lambda doc: parser(textacy.preprocess.replace_emails(doc.text,replace_with=replace_with))
    return lambda doc : parser(emailFinder.sub(replace_with, doc.text))

def replaceTwitterMentions(replace_with="TWITTER_MENTION",parser=PARSER):
    return lambda doc : parser(mentionFinder.sub(replace_with, doc.text))

def replaceNumbers(replace_with="NUMBER",parser=PARSER):
    return lambda doc: parser(textacy.preprocess.replace_numbers(doc.text, replace_with=replace_with))

def replacePhonenumbers(replace_with="PHONE",parser=PARSER):
    return lambda doc: parser(textacy.preprocess.replace_phone_numbers(doc.text, replace_with=replace_with))



def resolveAbbreviations(parser=PARSER):
    pass #todo

def removeWords(words, keep=None,parser=PARSER):
    if hasattr(keep, '__iter__'):
        for k in keep:
            try:
                words.remove(k)
            except ValueError:
                pass
    return lambda doc : parser(" ".join([tok.text for tok in doc if tok.lower_ not in words]))



def normalizeSynonyms(default_return_first_Syn=False, parser=PARSER):
    #return lambda doc : parser(" ".join([tok.lower_ for tok in doc]))
    return lambda doc : parser(" ".join([getFirstSynonym(tok.lower_, THESAURUS, default_return_first_Syn=default_return_first_Syn) for tok in doc]))

def getFirstSynonym(word, thesaurus, default_return_first_Syn=False):
    if not isinstance(word, str):
        return str(word)

    word = word.lower()

    # durch den thesaurrus iterieren
    for syn_block in thesaurus:  # syn_block ist eine liste mit Synonymen

        for syn in syn_block:
            syn = syn.lower()
            if re.match(r'\A[\w-]+\Z', syn):  # falls syn einzelwort ist
                if word == syn:
                    return str(getHauptform(syn_block, word, default_return_first_Syn=default_return_first_Syn))
            else:  # falls es ein satz ist
                if word in syn:
                    return str(getHauptform(syn_block, word, default_return_first_Syn=default_return_first_Syn))
    return str(word)  # zur Not, das ursrpüngliche Wort zurückgeben

def getHauptform(syn_block, word, default_return_first_Syn=False):
    for syn in syn_block:
        syn = syn.lower()

        if "hauptform" in syn and len(syn.split(" ")) <= 2:
            # nicht ausgeben, falls es in Klammern steht#todo gibts macnmal?? klammern aus
            for w in syn.split(" "):
                if not re.match(r'\([^)]+\)', w):
                    return w

    if default_return_first_Syn:
        # falls keine hauptform enthalten ist, das erste Synonym zurückgeben, was kein satz ist und nicht in klammern steht
        for w in syn_block:
            if not re.match(r'\([^)]+\)', w):
                return w
    return word  # zur Not, das ursrpüngliche Wort zurückgeben

def label2ID(label):
    return {
    'Neuanschluss' : 0,
    'LSF' : 1,
    'Video' : 2,
    }.get(label,3)

def generate_labled_lines(textacyCorpus):
    for doc in textacyCorpus:
        # generate [topic1, topic2....] tok1 tok2 tok3 out of corpus
        yield "[" + str(label2ID(doc.metadata["Kategorie"])) + "] " + doc.text




####################'####################'####################'####################'####################'##############



ents = config.get("preprocessing","ents").split(",")

clean_in_content=compose(

    doc2String,
    #normalizeSynonyms(default_return_first_Syn=config.get("preprocessing","default_return_first_Syn")),
    replaceEmails(),
    replaceURLs(),
    replaceTwitterMentions(),
    removeWords(stop_words),
    #removeAllPOS(["SPACE","PUNCT"]),
    #removeAllENT(ents),
    keepOnlyPOS(['NOUN'])
)

clean_in_meta = {
    "Loesung":removeAllPOS(["SPACE"]),
    "Zusammenfassung":removeAllPOS(["SPACE","PUNCT"])
}







## add files to textacy-corpus,
print("add texts to textacy-corpus...")
corpus.add_texts(
    processTextstream(generateMainTextfromTicketXML(path2xml), func=clean_in_content),
    processDictstream(generateMetadatafromTicketXML(path2xml), funcdict=clean_in_meta)
)

printRandomDoc(corpus)



####################'####################'      Variablen todo alles in config

ngrams = (1,2)

min_df = 0
max_df = 1.0
no_below = 20
no_above = 0.5

topicModel = 'lda'
# http://textacy.readthedocs.io/en/latest/api_reference.html#textacy.tm.topic_model.TopicModel.get_doc_topic_matrix
weighting = ('tf' if topicModel == 'lda' else 'tfidf')

top_topic_words = 5
top_document_labels_per_topic = 2

n_topics = 4






####################'####################




print("vectorize corpus...")
vectorizer = Vectorizer(weighting=weighting, min_df=min_df, max_df=max_df)

terms_list = (doc.to_terms_list(ngrams=ngrams, named_entities=False, as_strings=True) for doc in corpus)
doc_term_matrix = vectorizer.fit_transform(terms_list)
id2term = vectorizer.__getattribute__("id_to_term")











#####################     LSA, LDA, NMF         Topic Modeling via Textacy         ##############################################

# Initialize and train a topic model
print("Initialize and train a topic model")
model = textacy.tm.TopicModel(topicModel, n_topics=n_topics)
model.fit(doc_term_matrix)

#Transform the corpus and interpret our model:
print("Transform the corpus and interpret our model")
doc_topic_matrix = model.transform(doc_term_matrix)
print()


for topic_idx, top_terms in model.top_topic_terms(vectorizer.id_to_term, top_n=top_topic_words):
    print('topic', topic_idx, ':', '   '.join(top_terms))


print()
for topic_idx, top_docs in model.top_topic_docs(doc_topic_matrix, top_n=top_document_labels_per_topic):
     print(topic_idx)
     for j in top_docs:
        print(corpus[j].metadata['Kategorie'])

#####################################################################################################################
print()
print()





#####################   LLDA           Topic Modeling via JGibbsLabledLDA     ##############################################



jgibbsLLDA_root = "java_LabledLDA/"
filepath = "{0}models/tickets/tickets.gz".format(jgibbsLLDA_root)


#create file
textacy.fileio.write_file_lines(generate_labled_lines(corpus),filepath=filepath)


# wait for file to exist
while not os.path.exists(filepath):
    time.sleep(1)

print("start LLDA..")
#run JGibsslda file
FNULL = open(os.devnull, 'w') # supress output
subprocess.call(["java",
                 "-cp", "{0}lib/trove-3.0.3.jar:{0}lib/args4j-2.0.6.jar:{0}out/production/LabledLDA/".format(jgibbsLLDA_root),
                 "jgibblda.LDA",
                 "-est",
                 "-dir", "{0}models/tickets".format(jgibbsLLDA_root),
                 "-dfile","tickets.gz",
                 "-ntopics", str(n_topics)], stdout = FNULL)

# ANMERKUNG: Dateien sind versteckt. zu finden in models/

#print twords
subprocess.call(["gzip",
                 "-dc",
                 "{0}/models/tickets/.twords.gz".format(jgibbsLLDA_root)])
#####################################################################################################################
print()
print()