# -*- coding: utf-8 -*- from datetime import datetime import time import csv import sys import json import os.path import subprocess from textacy import Vectorizer from miscellaneous import * import textacy from scipy import * import os csv.field_size_limit(sys.maxsize) FILEPATH = os.path.dirname(os.path.realpath(__file__)) + "/" # ssh madonna "nohup /usr/bin/python3 -u /home/jannis.grundmann/PycharmProjects/topicModelingTickets/topicModeling.py &> /home/jannis.grundmann/PycharmProjects/topicModelingTickets/printout_topicModeling.log &" # load config config_ini = FILEPATH + "config.ini" config = ConfigParser.ConfigParser() with open(config_ini) as f: config.read_file(f) def printvecotorization(de_corpus,ngrams=1, min_df=1, max_df=1.0, weighting='tf', named_entities=True): printlog(str("ngrams: {0}".format(ngrams))) printlog(str("min_df: {0}".format(min_df))) printlog(str("max_df: {0}".format(max_df))) printlog(str("named_entities: {0}".format(named_entities))) # printlog("vectorize corpi...") vectorizer = Vectorizer(weighting=weighting, min_df=min_df, max_df=max_df) terms_list = (doc.to_terms_list(ngrams=ngrams, named_entities=named_entities, as_strings=True) for doc in de_corpus) doc_term_matrix = vectorizer.fit_transform(terms_list) id2term = vectorizer.__getattribute__("id_to_term") for t in terms_list: print(t) printlog("doc_term_matrix: {0}".format(doc_term_matrix)) printlog("id2term: {0}".format(id2term)) def textacyTopicModeling(ngrams, min_df, max_df, corpus, n_topics, topicModel='lda',named_entities=False): printlog( "############################################ Topic Modeling {0} #############################################".format( topicModel)) print("\n\n") printlog(str("ngrams: {0}".format(ngrams))) printlog(str("min_df: {0}".format(min_df))) printlog(str("max_df: {0}".format(max_df))) printlog(str("n_topics: {0}".format(n_topics))) printlog(str("named_entities: {0}".format(named_entities))) start = time.time() top_topic_words = 10 top_document_labels_per_topic = 5 # http://textacy.readthedocs.io/en/latest/api_reference.html#textacy.tm.topic_model.TopicModel.get_doc_topic_matrix weighting = ('tf' if topicModel == 'lda' else 'tfidf') ####################'#################### # printlog("vectorize corpi...") vectorizer = Vectorizer(weighting=weighting, min_df=min_df, max_df=max_df) terms_list = (doc.to_terms_list(ngrams=ngrams, named_entities=named_entities, as_strings=True) for doc in corpus) doc_term_matrix = vectorizer.fit_transform(terms_list) id2term = vectorizer.__getattribute__("id_to_term") # printlog("terms_list: {0}".format(list(terms_list))) # printlog("doc_term_matrix: {0}".format(doc_term_matrix)) ##################### LSA, LDA, NMF Topic Modeling via Textacy ############################################## # Initialize and train a topic model # printlog("Initialize and train a topic model..") model = textacy.tm.TopicModel(topicModel, n_topics=n_topics) model.fit(doc_term_matrix) # Transform the corpi and interpret our model: # printlog("Transform the corpi and interpret our model..") doc_topic_matrix = model.transform(doc_term_matrix) print() for topic_idx, top_terms in model.top_topic_terms(vectorizer.id_to_term, top_n=top_topic_words): printlog('topic {0}: {1}'.format(topic_idx, " ".join(top_terms))) print() for topic_idx, top_docs in model.top_topic_docs(doc_topic_matrix, top_n=top_document_labels_per_topic): printlog(topic_idx) for j in top_docs: printlog(corpus[j].metadata['categoryName']) print() ##################################################################################################################### print() print() end = time.time() printlog("\n\n\nTime Elapsed Topic Modeling with {1}:{0} min\n\n".format((end - start) / 60, topicModel)) def jgibbsLLDA(de_corpus, top_topic_words): ##################### LLDA Topic Modeling via JGibbsLabledLDA ############################################## start = time.time() def label2ID(label, labeldict): return labeldict.get(label, len(labeldict)) def generate_labled_lines(textacyCorpus,labeldict): for doc in textacyCorpus: # generate [topic1, topic2....] tok1 tok2 tok3 out of corpi yield "[" + str(label2ID(doc.metadata["categoryName"],labeldict)) + "] " + doc.text # build citionary of ticketcategories labelist = [] for texdoc in de_corpus.get(lambda texdoc: texdoc.metadata["categoryName"] not in labelist): labelist.append(texdoc.metadata["categoryName"]) labeldict = {k: v for v, k in enumerate(labelist)} n_topics = len(labeldict) + 1 # len(set(ticketcorpus[0].metadata.keys()))+1 #+1 wegen einem default-topic jgibbsLLDA_root = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/java_LabledLDA/" LLDA_filepath = "{0}models/tickets/tickets.gz".format(jgibbsLLDA_root) dict_path = "{0}models/tickets/labeldict.txt".format(jgibbsLLDA_root) #printlog(str("LABELDICT: {0}".format(labeldict))) printlog(str("LABELDICT-length: {0}".format(len(labeldict)))) with open(dict_path, 'w') as file: file.write(json.dumps(labeldict)) #for line in generate_labled_lines(de_corpus,labeldict): # print(line) # create file textacy.fileio.write_file_lines(generate_labled_lines(de_corpus,labeldict), filepath=LLDA_filepath) # wait for file to exist while not os.path.exists(LLDA_filepath): time.sleep(1) """ printlog("") printlog("start LLDA:") # run JGibsslda file FNULL = open(os.devnull, 'w') # supress output subprocess.call(["java", "-cp", "{0}lib/trove-3.0.3.jar:{0}lib/args4j-2.0.6.jar:{0}out/production/LabledLDA/".format( jgibbsLLDA_root), "jgibblda.LDA", "-est", "-dir", "{0}models/tickets".format(jgibbsLLDA_root), "-dfile", "tickets.gz", "-twords", str(top_topic_words), "-ntopics", str(n_topics)], stdout=FNULL) # ANMERKUNG: Dateien sind versteckt. zu finden in models/ # twords subprocess.call(["gzip", "-dc", "{0}/models/tickets/.twords.gz".format(jgibbsLLDA_root)]) ##################################################################################################################### printlog("") """ end = time.time() printlog("\n\n\nTime Elapsed Topic Modeling JGibbsLLDA:{0} min\n\n".format((end - start) / 60)) def main(): printlog("Topic Modeling: {0}".format(datetime.now())) corpus_de_path = FILEPATH + config.get("de_corpus", "path") corpus_en_path = FILEPATH + config.get("en_corpus", "path") preCorpus_name = "de" + "_pre_ticket" #load raw corpus and create new one de_corpus, parser = load_corpus(corpus_name=preCorpus_name, corpus_path=corpus_de_path) printlog("Corpus loaded: {0}".format(de_corpus.lang)) #idee http://bigartm.org/ #idee http://wiki.languagetool.org/tips-and-tricks # todo gescheites tf(-idf) maß finden ngrams = 1 min_df = 1 max_df = 1.0 weighting = 'tf' # weighting ='tfidf' named_entities = False """ printvecotorization(ngrams=1, min_df=1, max_df=1.0, weighting=weighting) printvecotorization(ngrams=1, min_df=1, max_df=0.5, weighting=weighting) printvecotorization(ngrams=1, min_df=1, max_df=0.8, weighting=weighting) printvecotorization(ngrams=(1, 2), min_df=1, max_df=1.0, weighting=weighting) printvecotorization(ngrams=(1, 2), min_df=1, max_df=0.5, weighting=weighting) printvecotorization(ngrams=(1, 2), min_df=1, max_df=0.8, weighting=weighting) """ jgibbsLLDA(de_corpus,15) # no_below = 20 # no_above = 0.5 # n_topics = len(LABELDICT)#len(set(ticketcorpus[0].metadata.keys()))+1 #+1 wegen einem default-topic """ topicModeling(ngrams = 1, min_df = 1, max_df = 1.0, topicModel = 'lda', n_topics = len(LABELDICT), corpi=de_corpus) topicModeling(ngrams = 1, min_df = 0.1, max_df = 0.6, topicModel = 'lda', n_topics = len(LABELDICT), corpi=de_corpus) topicModeling(ngrams = (1,2), min_df = 1, max_df = 1.0, topicModel = 'lda', n_topics = len(LABELDICT), corpi=de_corpus) topicModeling(ngrams = (1,2), min_df = 0.1, max_df = 0.6, topicModel = 'lda', n_topics = len(LABELDICT), corpi=de_corpus) topicModeling(ngrams = (1,2), min_df = 0.2, max_df = 0.8, topicModel = 'lda', n_topics = 20, corpi=de_corpus) """ if __name__ == "__main__": main()