topicModelingTickets/topicModeling.py

# -*- coding: utf-8 -*-

from datetime import datetime

import time

import csv
import sys
import json
import os.path
import subprocess
from textacy import Vectorizer

from miscellaneous import *
import textacy
from scipy import *

import os

csv.field_size_limit(sys.maxsize)
FILEPATH = os.path.dirname(os.path.realpath(__file__)) + "/"

# ssh madonna "nohup /usr/bin/python3 -u /home/jannis.grundmann/PycharmProjects/topicModelingTickets/topicModeling.py &> /home/jannis.grundmann/PycharmProjects/topicModelingTickets/printout_topicModeling.log &"


# load config
config_ini = FILEPATH + "config.ini"

config = ConfigParser.ConfigParser()
with open(config_ini) as f:
    config.read_file(f)




def printvecotorization(de_corpus,ngrams=1, min_df=1, max_df=1.0, weighting='tf', named_entities=True):
    printlog(str("ngrams: {0}".format(ngrams)))
    printlog(str("min_df: {0}".format(min_df)))
    printlog(str("max_df: {0}".format(max_df)))
    printlog(str("named_entities: {0}".format(named_entities)))

    # printlog("vectorize corpi...")
    vectorizer = Vectorizer(weighting=weighting, min_df=min_df, max_df=max_df)

    terms_list = (doc.to_terms_list(ngrams=ngrams, named_entities=named_entities, as_strings=True) for doc in de_corpus)
    doc_term_matrix = vectorizer.fit_transform(terms_list)
    id2term = vectorizer.__getattribute__("id_to_term")

    for t in terms_list:
        print(t)
    printlog("doc_term_matrix: {0}".format(doc_term_matrix))
    printlog("id2term: {0}".format(id2term))

def textacyTopicModeling(ngrams, min_df, max_df, corpus, n_topics,  topicModel='lda',named_entities=False):
    printlog(
        "############################################ Topic Modeling {0}   #############################################".format(
            topicModel))
    print("\n\n")
    printlog(str("ngrams: {0}".format(ngrams)))
    printlog(str("min_df: {0}".format(min_df)))
    printlog(str("max_df: {0}".format(max_df)))
    printlog(str("n_topics: {0}".format(n_topics)))
    printlog(str("named_entities: {0}".format(named_entities)))

    start = time.time()

    top_topic_words = 10
    top_document_labels_per_topic = 5

    # http://textacy.readthedocs.io/en/latest/api_reference.html#textacy.tm.topic_model.TopicModel.get_doc_topic_matrix
    weighting = ('tf' if topicModel == 'lda' else 'tfidf')

    ####################'####################


    # printlog("vectorize corpi...")
    vectorizer = Vectorizer(weighting=weighting, min_df=min_df, max_df=max_df)

    terms_list = (doc.to_terms_list(ngrams=ngrams, named_entities=named_entities, as_strings=True) for doc in corpus)
    doc_term_matrix = vectorizer.fit_transform(terms_list)
    id2term = vectorizer.__getattribute__("id_to_term")

    # printlog("terms_list: {0}".format(list(terms_list)))
    # printlog("doc_term_matrix: {0}".format(doc_term_matrix))



    #####################     LSA, LDA, NMF         Topic Modeling via Textacy         ##############################################

    # Initialize and train a topic model
    # printlog("Initialize and train a topic model..")
    model = textacy.tm.TopicModel(topicModel, n_topics=n_topics)
    model.fit(doc_term_matrix)

    # Transform the corpi and interpret our model:
    # printlog("Transform the corpi and interpret our model..")
    doc_topic_matrix = model.transform(doc_term_matrix)
    print()

    for topic_idx, top_terms in model.top_topic_terms(vectorizer.id_to_term, top_n=top_topic_words):
        printlog('topic {0}: {1}'.format(topic_idx, " ".join(top_terms)))

    print()
    for topic_idx, top_docs in model.top_topic_docs(doc_topic_matrix, top_n=top_document_labels_per_topic):
        printlog(topic_idx)
        for j in top_docs:
            printlog(corpus[j].metadata['categoryName'])
        print()

    #####################################################################################################################
    print()
    print()

    end = time.time()
    printlog("\n\n\nTime Elapsed Topic Modeling with {1}:{0} min\n\n".format((end - start) / 60, topicModel))


def jgibbsLLDA(de_corpus, top_topic_words):
    #####################   LLDA           Topic Modeling via JGibbsLabledLDA     ##############################################

    start = time.time()

    def label2ID(label, labeldict):
        return labeldict.get(label, len(labeldict))

    def generate_labled_lines(textacyCorpus,labeldict):
        for doc in textacyCorpus:
            # generate [topic1, topic2....] tok1 tok2 tok3 out of corpi
            yield "[" + str(label2ID(doc.metadata["categoryName"],labeldict)) + "] " + doc.text

    # build citionary of ticketcategories
    labelist = []

    for texdoc in de_corpus.get(lambda texdoc: texdoc.metadata["categoryName"] not in labelist):
        labelist.append(texdoc.metadata["categoryName"])

    labeldict = {k: v for v, k in enumerate(labelist)}

    n_topics = len(labeldict) + 1  # len(set(ticketcorpus[0].metadata.keys()))+1 #+1 wegen einem default-topic

    jgibbsLLDA_root = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/java_LabledLDA/"

    LLDA_filepath = "{0}models/tickets/tickets.gz".format(jgibbsLLDA_root)
    dict_path = "{0}models/tickets/labeldict.txt".format(jgibbsLLDA_root)


    #printlog(str("LABELDICT: {0}".format(labeldict)))
    printlog(str("LABELDICT-length: {0}".format(len(labeldict))))
    with open(dict_path, 'w') as file:
        file.write(json.dumps(labeldict))

    #for line in generate_labled_lines(de_corpus,labeldict):
    #    print(line)

    # create file
    textacy.fileio.write_file_lines(generate_labled_lines(de_corpus,labeldict), filepath=LLDA_filepath)

    # wait for file to exist
    while not os.path.exists(LLDA_filepath):
        time.sleep(1)
    """
    printlog("")
    printlog("start LLDA:")
    # run JGibsslda file
    FNULL = open(os.devnull, 'w')  # supress output
    subprocess.call(["java",
                     "-cp",
                     "{0}lib/trove-3.0.3.jar:{0}lib/args4j-2.0.6.jar:{0}out/production/LabledLDA/".format(
                         jgibbsLLDA_root),
                     "jgibblda.LDA",
                     "-est",
                     "-dir", "{0}models/tickets".format(jgibbsLLDA_root),
                     "-dfile", "tickets.gz",
                     "-twords", str(top_topic_words),
                     "-ntopics", str(n_topics)], stdout=FNULL)

    # ANMERKUNG: Dateien sind versteckt. zu finden in models/

    # twords
    subprocess.call(["gzip",
                     "-dc",
                     "{0}/models/tickets/.twords.gz".format(jgibbsLLDA_root)])
    #####################################################################################################################
    printlog("")
    """
    end = time.time()
    printlog("\n\n\nTime Elapsed Topic Modeling JGibbsLLDA:{0} min\n\n".format((end - start) / 60))



def main():

    printlog("Topic Modeling: {0}".format(datetime.now()))

    corpus_de_path = FILEPATH + config.get("de_corpus", "path")

    corpus_en_path = FILEPATH + config.get("en_corpus", "path")

    preCorpus_name = "de" + "_pre_ticket"

    #load raw corpus and create new one
    de_corpus, parser = load_corpus(corpus_name=preCorpus_name, corpus_path=corpus_de_path)
    printlog("Corpus loaded: {0}".format(de_corpus.lang))

    #idee http://bigartm.org/
    #idee http://wiki.languagetool.org/tips-and-tricks

    # todo gescheites tf(-idf) maß finden
    ngrams = 1
    min_df = 1
    max_df = 1.0
    weighting = 'tf'
    # weighting ='tfidf'
    named_entities = False


    """
    printvecotorization(ngrams=1, min_df=1, max_df=1.0, weighting=weighting)
    printvecotorization(ngrams=1, min_df=1, max_df=0.5, weighting=weighting)
    printvecotorization(ngrams=1, min_df=1, max_df=0.8, weighting=weighting)
    
    printvecotorization(ngrams=(1, 2), min_df=1, max_df=1.0, weighting=weighting)
    printvecotorization(ngrams=(1, 2), min_df=1, max_df=0.5, weighting=weighting)
    printvecotorization(ngrams=(1, 2), min_df=1, max_df=0.8, weighting=weighting)
    """


    jgibbsLLDA(de_corpus,15)

    # no_below = 20
    # no_above = 0.5


    # n_topics = len(LABELDICT)#len(set(ticketcorpus[0].metadata.keys()))+1 #+1 wegen einem default-topic



    """
    topicModeling(ngrams = 1,
                  min_df = 1,
                  max_df = 1.0,
                  topicModel = 'lda',
                  n_topics = len(LABELDICT),
                  corpi=de_corpus)
    
    topicModeling(ngrams = 1,
                  min_df = 0.1,
                  max_df = 0.6,
                  topicModel = 'lda',
                  n_topics = len(LABELDICT),
                  corpi=de_corpus)
    
    topicModeling(ngrams = (1,2),
                  min_df = 1,
                  max_df = 1.0,
                  topicModel = 'lda',
                  n_topics = len(LABELDICT),
                  corpi=de_corpus)
    
    topicModeling(ngrams = (1,2),
                  min_df = 0.1,
                  max_df = 0.6,
                  topicModel = 'lda',
                  n_topics = len(LABELDICT),
                  corpi=de_corpus)
    
    topicModeling(ngrams = (1,2),
                  min_df = 0.2,
                  max_df = 0.8,
                  topicModel = 'lda',
                  n_topics = 20,
                  corpi=de_corpus)
    
    
    
    
    
    
    
    """



if __name__ == "__main__":
    main()