topicModelingTickets/topicModeling.py

# -*- coding: utf-8 -*-

from datetime import datetime

import time
import numpy as np

import csv
import sys
import json
import os.path
import subprocess
from textacy import Vectorizer, viz

from miscellaneous import *
import textacy
from scipy import *

import os

csv.field_size_limit(sys.maxsize)
FILEPATH = os.path.dirname(os.path.realpath(__file__)) + "/"


# load config
config_ini = FILEPATH + "config.ini"

config = ConfigParser.ConfigParser()
with open(config_ini) as f:
    config.read_file(f)


def label2ID(label, labeldict):
    return labeldict.get(label, len(labeldict))


def generate_lablelID_lines(textacyCorpus, labeldict):
    for doc in textacyCorpus:
        # generate [topic1, topic2....] tok1 tok2 tok3 out of corpi
        yield "[" + str(label2ID(doc.metadata["categoryName"], labeldict)) + "] " + doc.text

"""
def printvecotorization(de_corpus, ngrams=1, min_df=1, max_df=1.0, weighting='tf', named_entities=True):
    logprint(str("ngrams: {0}".format(ngrams)))
    logprint(str("min_df: {0}".format(min_df)))
    logprint(str("max_df: {0}".format(max_df)))
    logprint(str("named_entities: {0}".format(named_entities)))

    # printlog("vectorize corpi...")
    vectorizer = Vectorizer(weighting=weighting, min_df=min_df, max_df=max_df)

    terms_list = (doc.to_terms_list(ngrams=ngrams, named_entities=named_entities, as_strings=True) for doc in de_corpus)
    doc_term_matrix = vectorizer.fit_transform(terms_list)
    id2term = vectorizer.__getattribute__("id_to_term")

    for t in terms_list:
        print(t)
    logprint("doc_term_matrix: {0}".format(doc_term_matrix))
    logprint("id2term: {0}".format(id2term))
"""

def textacyTopicModeling(corpus,
                         n_topics = 15, top_topic_words = 7, top_document_labels_per_topic = 5,
                         ngrams = 1, min_df=1, max_df=1.0,
                         topicModel='lda'):




    n_terms = int(n_topics * top_topic_words)
    sort_terms_by = 'seriation'  # 'seriation', 'weight', 'index', 'alphabetical'
    rank_terms_by = 'corpus'  # 'corpus', 'topic'




    logprint(
        "############### Topic Modeling {0}   ###########################".format(
            topicModel))
    logprint(str("ngrams: {0}".format(ngrams)))
    logprint(str("min_df: {0}".format(min_df)))
    logprint(str("max_df: {0}".format(max_df)))
    logprint(str("n_topics: {0}".format(n_topics)))
    logprint("\n")

    start = time.time()

    # http://textacy.readthedocs.io/en/latest/api_reference.html#textacy.tm.topic_model.TopicModel.get_doc_topic_matrix
    weighting = ('tf' if topicModel == 'lda' else 'tfidf')






    ####################    vectorize corpi     ####################

    vectorizer = Vectorizer(weighting=weighting, min_df=min_df, max_df=max_df)

    terms_list = (doc.to_terms_list(ngrams=ngrams, named_entities=False, as_strings=True) for doc in corpus)
    doc_term_matrix = vectorizer.fit_transform(terms_list)
    id2term = vectorizer.__getattribute__("id_to_term")

    # printlog("terms_list: {0}".format(list(terms_list)))
    # printlog("doc_term_matrix: {0}".format(doc_term_matrix))





    #####################   Initialize and train a topic model        ##############################################

    model = textacy.tm.TopicModel(topicModel, n_topics=n_topics)

    model.fit(doc_term_matrix)

    doc_topic_matrix = model.transform(doc_term_matrix)


    for topic_idx, top_terms in model.top_topic_terms(vectorizer.id_to_term, top_n=top_topic_words):
        logprint('topic {0}: {1}'.format(topic_idx, " ".join(top_terms)))

    for topic_idx, top_docs in model.top_topic_docs(doc_topic_matrix, top_n=top_document_labels_per_topic):
        logprint(topic_idx)
        for j in top_docs:
            logprint(corpus[j].metadata['categoryName'])




    #######################  termite plot        ###################################################################

    grams_label = "uni" if ngrams == 1 else "bi"

    model.termite_plot(doc_term_matrix, id2term,

                       n_terms=n_terms,
                       sort_terms_by=sort_terms_by,
                       rank_terms_by=rank_terms_by+'_weight',


    save= FILEPATH + "results/{}_{}_{}_{}_{}_{}.png".format(grams_label,topicModel,n_topics,n_terms,sort_terms_by,rank_terms_by))



    end = time.time()
    logprint("\n\n\nTime Elapsed Topic Modeling with {1}:{0} min\n\n".format((end - start) / 60, topicModel))



def jgibbsLLDA(corpus, path2save_results, top_topic_words=7):
    start = time.time()



    jgibbsLLDA_root = FILEPATH + "java_LabledLDA/"

    LLDA_filepath = "{0}models/tickets/tickets.gz".format(jgibbsLLDA_root)




    # build  dictionary of ticketcategories
    labelist = []
    for texdoc in corpus.get(lambda texdoc: texdoc.metadata["categoryName"] not in labelist):
        labelist.append(texdoc.metadata["categoryName"])


    labeldict = {k: v for v, k in enumerate(labelist)}
    reverse_labeldict = {v: k for k, v in labeldict.items()}

    #and save
    labeldict_path = FILEPATH + "results/labeldict.txt"
    with open(labeldict_path, 'w') as file:
        file.write(json.dumps(labeldict))


    n_topics = len(labeldict)  #+1  #default-topic



    # create file with label_IDs (input for llda)
    textacy.fileio.write_file_lines(generate_lablelID_lines(corpus, labeldict), filepath=LLDA_filepath)

    # wait for file to exist
    while not os.path.exists(LLDA_filepath):
        time.sleep(1)

    logprint("")
    logprint("start LLDA:")


    # run JGibbsLLDA file

    FNULL = open(os.devnull, 'w')  # supress output
    cmd_jgibbs_java = ["java", "-cp",
                       "{0}lib/trove-3.0.3.jar:{0}lib/args4j-2.0.6.jar:{0}out/production/LabledLDA/".format(
                           jgibbsLLDA_root),
                       "jgibblda.LDA", "-est", "-dir", "{0}models/tickets".format(jgibbsLLDA_root), "-dfile",
                       "tickets.gz",
                       "-twords", str(top_topic_words), "-ntopics", str(n_topics)]
    subprocess.call(cmd_jgibbs_java, stdout=FNULL)


    # ANMERKUNG: Dateien sind versteckt. zu finden in models/
    cmd_gzip = ["gzip", "-dc", "{0}/models/tickets/.twords.gz".format(jgibbsLLDA_root)]
    output = subprocess.check_output(cmd_gzip).decode("utf-8")


    topic_regex = re.compile(r'Topic [0-9]*')

    #####################################
    # todo save results in file aufgrund von results
    result = []

    for line in output.splitlines():
        findall = topic_regex.findall(line)
        if len(findall) != 0:
            try:
                index = int(findall[0].split()[1])
                result.append("Topic {} {}:".format(index, reverse_labeldict[index]))

            except:
                result.append(line)

        else:
            result.append(line)

    textacy.fileio.write_file_lines(result, path2save_results+".txt")
    #####################################

    results = []
    res_dict = {}
    count =0
    for line in output.splitlines():

        findall = topic_regex.findall(line)

        if len(findall) != 0:

            if len(res_dict) != 0:
                results.append(res_dict)    #vorheriges an die liste ran (ist ja dann fertig)

            index = int(findall[0].split()[1])

            res_dict = {index : str(reverse_labeldict[index]) }

        else:
            splitted = line.split()
            res_dict[splitted[0]] = float(splitted[1])

            ### print terms that are topics
            for s  in list(res_dict.values()):
                if isinstance(s,str) and splitted[0] in s:
                    vals = list(res_dict.values())
                    keys = list(res_dict.keys())
                    for v in vals:
                        if not isinstance(v,float):
                            print("{}".format(v))
                    print("{}".format(splitted[0]))
                    count +=1
                    print()
            ###


    if len(res_dict) != 0:
        results.append(res_dict)  # letzes an die liste ran

    print(count)
    print(float(count)/float(len(labelist)))




    # {0: 'betrieb', 'service': 0.24162679425837305, 'support': 0.24162679425837305, 'browser': 0.24162679425837305, 'unicard': 0.24162679425837305, 'telefon': 0.0023923444976076593}


    # every term in the resulsts to a list

    terms=[]
    for res in results:
        for key,value in res.items():
            if not isinstance(key, int) and not key in terms:
                terms.append(key)

    term2id = {t:i for i,t in enumerate(terms)} #and to dict

    #################   termite plot    #####################################################################

    #term_topic_weights.shape = (len(term_ids),len(topic_ids)


    #topic_labels = tuple(labelist)

    topic_labels = list(range(len(labelist)))
    term_labels = list(range(len(term2id))) #tuple([key for key in term2id.keys()])


    term_topic_weights = np.zeros((len(term2id),len(topic_labels)))



    for i,res in enumerate(results):

        for key,value in res.items():

            if not isinstance(key, int):
                term_topic_weights[term2id[key]][i] = value
                term_labels[term2id[key]] = key
            else:
                topic_labels[i] = reverse_labeldict[key]


    viz.draw_termite_plot(
        term_topic_weights, topic_labels, term_labels, save=path2save_results+".png")


    end = time.time()
    logprint("Time Elapsed Topic Modeling JGibbsLLDA:{0} min\n".format((end - start) / 60))





def main(use_cleaned=False, algorithm="llda"):


    # idee http://bigartm.org/
    # idee http://wiki.languagetool.org/tips-and-tricks
    # idee https://en.wikipedia.org/wiki/Noisy_text_analytics
    # idee https://gate.ac.uk/family/



    logprint("Topic Modeling: {0}".format(datetime.now()))

    corpus_de_path = FILEPATH + config.get("de_corpus", "path")
    corpus_en_path = FILEPATH + config.get("en_corpus", "path")


    if use_cleaned:
        preCorpus_name = "de" + "_clean_ticket"
        resultspath = FILEPATH + "results/clean"
    else:
        preCorpus_name = "de" + "_pre_ticket"
        resultspath = FILEPATH + "results/pre"



    # load cleand corpus
    de_corpus, parser = load_corpus(corpus_name=preCorpus_name, corpus_path=corpus_de_path)
    logprint("Corpus loaded: {0}".format(de_corpus.lang))



    # todo llda topics zusammenfassen
    # idee lda so trainieren, dass zuordnung term <-> topic nicht zu schwach wird, aber möglichst viele topics
    # frage wieviele tickets pro topic?

    """
    ngrams = 1
    min_df = 1
    max_df = 1.0
    weighting = 'tf'
    # weighting ='tfidf'
    named_entities = False

  
    printvecotorization(ngrams=1, min_df=1, max_df=1.0, weighting=weighting)
    printvecotorization(ngrams=1, min_df=1, max_df=0.5, weighting=weighting)
    printvecotorization(ngrams=1, min_df=1, max_df=0.8, weighting=weighting)

    printvecotorization(ngrams=(1, 2), min_df=1, max_df=1.0, weighting=weighting)
    printvecotorization(ngrams=(1, 2), min_df=1, max_df=0.5, weighting=weighting)
    printvecotorization(ngrams=(1, 2), min_df=1, max_df=0.8, weighting=weighting)
    """

    if algorithm == "llda":
        top_topic_words = 5
        path2save_results = resultspath +  "_{}_{}".format(algorithm,"top"+str(top_topic_words))
        jgibbsLLDA(de_corpus, path2save_results=path2save_results, top_topic_words=top_topic_words)

        """
        top_topic_words = 10
        path2save_results = resultspath +  "_{}_{}".format(algorithm,"top"+str(top_topic_words))
        jgibbsLLDA(de_corpus, path2save_results=path2save_results, top_topic_words=top_topic_words)


        top_topic_words = 15
        path2save_results = resultspath + "_{}_{}".format(algorithm, "top" + str(top_topic_words))
        jgibbsLLDA(de_corpus, path2save_results=path2save_results, top_topic_words=top_topic_words)

        top_topic_words = 20
        path2save_results = resultspath + "_{}_{}".format(algorithm, "top" + str(top_topic_words))
        jgibbsLLDA(de_corpus, path2save_results=path2save_results, top_topic_words=top_topic_words)

        """
    else:


        textacyTopicModeling(ngrams = 1,
                             min_df = 1,
                             max_df = 0.9,
                             topicModel = algorithm,
                             n_topics =15,
                             corpus=de_corpus)
        """
        textacyTopicModeling(ngrams=1,
                             min_df=1,
                             max_df=0.9,
                             topicModel=algorithm,
                             n_topics=20,
                             corpus=de_corpus)

        textacyTopicModeling(ngrams=1,
                             min_df=1,
                             max_df=0.9,
                             topicModel=algorithm,
                             n_topics=25,
                             corpus=de_corpus)


        textacyTopicModeling(ngrams=1,
                             min_df=1,
                             max_df=0.9,
                             topicModel=algorithm,
                             n_topics=30,
                             corpus=de_corpus)
        """


        textacyTopicModeling(ngrams=(1, 2),
                             min_df=1,
                             max_df=0.9,
                             topicModel=algorithm,
                             n_topics=15,
                             corpus=de_corpus)
        """
        textacyTopicModeling(ngrams = (1,2),
                             min_df = 1,
                             max_df = 0.9,
                             topicModel = algorithm,
                             n_topics =20,
                             corpus=de_corpus)

        textacyTopicModeling(ngrams = (1,2),
                             min_df = 1,
                             max_df = 0.9,
                             topicModel = algorithm,
                             n_topics =25,
                             corpus=de_corpus)


        textacyTopicModeling(ngrams = (1,2),
                             min_df = 1,
                             max_df = 0.9,
                             topicModel = algorithm,
                             n_topics =30,
                             corpus=de_corpus)
        """




if __name__ == "__main__":
    main()