# -*- coding: utf-8 -*- from datetime import datetime import time import numpy as np import csv import sys import json import os.path import subprocess from textacy import Vectorizer, viz from miscellaneous import * import textacy from scipy import * import os csv.field_size_limit(sys.maxsize) FILEPATH = os.path.dirname(os.path.realpath(__file__)) + "/" # load config config_ini = FILEPATH + "config.ini" config = ConfigParser.ConfigParser() with open(config_ini) as f: config.read_file(f) def label2ID(label, labeldict): return labeldict.get(label, len(labeldict)) def generate_lablelID_lines(textacyCorpus, labeldict): for doc in textacyCorpus: # generate [topic1, topic2....] tok1 tok2 tok3 out of corpi yield "[" + str(label2ID(doc.metadata["categoryName"], labeldict)) + "] " + doc.text """ def printvecotorization(de_corpus, ngrams=1, min_df=1, max_df=1.0, weighting='tf', named_entities=True): logprint(str("ngrams: {0}".format(ngrams))) logprint(str("min_df: {0}".format(min_df))) logprint(str("max_df: {0}".format(max_df))) logprint(str("named_entities: {0}".format(named_entities))) # printlog("vectorize corpi...") vectorizer = Vectorizer(weighting=weighting, min_df=min_df, max_df=max_df) terms_list = (doc.to_terms_list(ngrams=ngrams, named_entities=named_entities, as_strings=True) for doc in de_corpus) doc_term_matrix = vectorizer.fit_transform(terms_list) id2term = vectorizer.__getattribute__("id_to_term") for t in terms_list: print(t) logprint("doc_term_matrix: {0}".format(doc_term_matrix)) logprint("id2term: {0}".format(id2term)) """ def textacyTopicModeling(corpus, n_topics = 15, top_topic_words = 7, top_document_labels_per_topic = 5, ngrams = 1, min_df=1, max_df=1.0, topicModel='lda'): n_terms = int(n_topics * top_topic_words) sort_terms_by = 'seriation' # 'seriation', 'weight', 'index', 'alphabetical' rank_terms_by = 'corpus' # 'corpus', 'topic' logprint( "############### Topic Modeling {0} ###########################".format( topicModel)) logprint(str("ngrams: {0}".format(ngrams))) logprint(str("min_df: {0}".format(min_df))) logprint(str("max_df: {0}".format(max_df))) logprint(str("n_topics: {0}".format(n_topics))) logprint("\n") start = time.time() # http://textacy.readthedocs.io/en/latest/api_reference.html#textacy.tm.topic_model.TopicModel.get_doc_topic_matrix weighting = ('tf' if topicModel == 'lda' else 'tfidf') #################### vectorize corpi #################### vectorizer = Vectorizer(weighting=weighting, min_df=min_df, max_df=max_df) terms_list = (doc.to_terms_list(ngrams=ngrams, named_entities=False, as_strings=True) for doc in corpus) doc_term_matrix = vectorizer.fit_transform(terms_list) id2term = vectorizer.__getattribute__("id_to_term") # printlog("terms_list: {0}".format(list(terms_list))) # printlog("doc_term_matrix: {0}".format(doc_term_matrix)) ##################### Initialize and train a topic model ############################################## model = textacy.tm.TopicModel(topicModel, n_topics=n_topics) model.fit(doc_term_matrix) doc_topic_matrix = model.transform(doc_term_matrix) for topic_idx, top_terms in model.top_topic_terms(vectorizer.id_to_term, top_n=top_topic_words): logprint('topic {0}: {1}'.format(topic_idx, " ".join(top_terms))) for topic_idx, top_docs in model.top_topic_docs(doc_topic_matrix, top_n=top_document_labels_per_topic): logprint(topic_idx) for j in top_docs: logprint(corpus[j].metadata['categoryName']) ####################### termite plot ################################################################### grams_label = "uni" if ngrams == 1 else "bi" model.termite_plot(doc_term_matrix, id2term, n_terms=n_terms, sort_terms_by=sort_terms_by, rank_terms_by=rank_terms_by+'_weight', save= FILEPATH + "results/{}_{}_{}_{}_{}_{}.png".format(grams_label,topicModel,n_topics,n_terms,sort_terms_by,rank_terms_by)) end = time.time() logprint("\n\n\nTime Elapsed Topic Modeling with {1}:{0} min\n\n".format((end - start) / 60, topicModel)) def jgibbsLLDA(corpus, path2save_results, top_topic_words=7): start = time.time() jgibbsLLDA_root = FILEPATH + "java_LabledLDA/" LLDA_filepath = "{0}models/tickets/tickets.gz".format(jgibbsLLDA_root) # build dictionary of ticketcategories labelist = [] for texdoc in corpus.get(lambda texdoc: texdoc.metadata["categoryName"] not in labelist): labelist.append(texdoc.metadata["categoryName"]) labeldict = {k: v for v, k in enumerate(labelist)} reverse_labeldict = {v: k for k, v in labeldict.items()} #and save labeldict_path = FILEPATH + "results/labeldict.txt" with open(labeldict_path, 'w') as file: file.write(json.dumps(labeldict)) n_topics = len(labeldict) #+1 #default-topic # create file with label_IDs (input for llda) textacy.fileio.write_file_lines(generate_lablelID_lines(corpus, labeldict), filepath=LLDA_filepath) # wait for file to exist while not os.path.exists(LLDA_filepath): time.sleep(1) logprint("") logprint("start LLDA:") # run JGibbsLLDA file FNULL = open(os.devnull, 'w') # supress output cmd_jgibbs_java = ["java", "-cp", "{0}lib/trove-3.0.3.jar:{0}lib/args4j-2.0.6.jar:{0}out/production/LabledLDA/".format( jgibbsLLDA_root), "jgibblda.LDA", "-est", "-dir", "{0}models/tickets".format(jgibbsLLDA_root), "-dfile", "tickets.gz", "-twords", str(top_topic_words), "-ntopics", str(n_topics)] subprocess.call(cmd_jgibbs_java, stdout=FNULL) # ANMERKUNG: Dateien sind versteckt. zu finden in models/ cmd_gzip = ["gzip", "-dc", "{0}/models/tickets/.twords.gz".format(jgibbsLLDA_root)] output = subprocess.check_output(cmd_gzip).decode("utf-8") topic_regex = re.compile(r'Topic [0-9]*') ##################################### # todo save results in file aufgrund von results result = [] for line in output.splitlines(): findall = topic_regex.findall(line) if len(findall) != 0: try: index = int(findall[0].split()[1]) result.append("Topic {} {}:".format(index, reverse_labeldict[index])) except: result.append(line) else: result.append(line) textacy.fileio.write_file_lines(result, path2save_results+".txt") ##################################### results = [] res_dict = {} count =0 for line in output.splitlines(): findall = topic_regex.findall(line) if len(findall) != 0: if len(res_dict) != 0: results.append(res_dict) #vorheriges an die liste ran (ist ja dann fertig) index = int(findall[0].split()[1]) res_dict = {index : str(reverse_labeldict[index]) } else: splitted = line.split() res_dict[splitted[0]] = float(splitted[1]) ### print terms that are topics for s in list(res_dict.values()): if isinstance(s,str) and splitted[0] in s: vals = list(res_dict.values()) keys = list(res_dict.keys()) for v in vals: if not isinstance(v,float): print("{}".format(v)) print("{}".format(splitted[0])) count +=1 print() ### if len(res_dict) != 0: results.append(res_dict) # letzes an die liste ran print(count) print(float(count)/float(len(labelist))) # {0: 'betrieb', 'service': 0.24162679425837305, 'support': 0.24162679425837305, 'browser': 0.24162679425837305, 'unicard': 0.24162679425837305, 'telefon': 0.0023923444976076593} # every term in the resulsts to a list terms=[] for res in results: for key,value in res.items(): if not isinstance(key, int) and not key in terms: terms.append(key) term2id = {t:i for i,t in enumerate(terms)} #and to dict ################# termite plot ##################################################################### #term_topic_weights.shape = (len(term_ids),len(topic_ids) #topic_labels = tuple(labelist) topic_labels = list(range(len(labelist))) term_labels = list(range(len(term2id))) #tuple([key for key in term2id.keys()]) term_topic_weights = np.zeros((len(term2id),len(topic_labels))) for i,res in enumerate(results): for key,value in res.items(): if not isinstance(key, int): term_topic_weights[term2id[key]][i] = value term_labels[term2id[key]] = key else: topic_labels[i] = reverse_labeldict[key] viz.draw_termite_plot( term_topic_weights, topic_labels, term_labels, save=path2save_results+".png") end = time.time() logprint("Time Elapsed Topic Modeling JGibbsLLDA:{0} min\n".format((end - start) / 60)) def main(use_cleaned=False, algorithm="llda"): # idee http://bigartm.org/ # idee http://wiki.languagetool.org/tips-and-tricks # idee https://en.wikipedia.org/wiki/Noisy_text_analytics # idee https://gate.ac.uk/family/ logprint("Topic Modeling: {0}".format(datetime.now())) corpus_de_path = FILEPATH + config.get("de_corpus", "path") corpus_en_path = FILEPATH + config.get("en_corpus", "path") if use_cleaned: preCorpus_name = "de" + "_clean_ticket" resultspath = FILEPATH + "results/clean" else: preCorpus_name = "de" + "_pre_ticket" resultspath = FILEPATH + "results/pre" # load cleand corpus de_corpus, parser = load_corpus(corpus_name=preCorpus_name, corpus_path=corpus_de_path) logprint("Corpus loaded: {0}".format(de_corpus.lang)) # todo llda topics zusammenfassen # idee lda so trainieren, dass zuordnung term <-> topic nicht zu schwach wird, aber möglichst viele topics # frage wieviele tickets pro topic? """ ngrams = 1 min_df = 1 max_df = 1.0 weighting = 'tf' # weighting ='tfidf' named_entities = False printvecotorization(ngrams=1, min_df=1, max_df=1.0, weighting=weighting) printvecotorization(ngrams=1, min_df=1, max_df=0.5, weighting=weighting) printvecotorization(ngrams=1, min_df=1, max_df=0.8, weighting=weighting) printvecotorization(ngrams=(1, 2), min_df=1, max_df=1.0, weighting=weighting) printvecotorization(ngrams=(1, 2), min_df=1, max_df=0.5, weighting=weighting) printvecotorization(ngrams=(1, 2), min_df=1, max_df=0.8, weighting=weighting) """ if algorithm == "llda": top_topic_words = 5 path2save_results = resultspath + "_{}_{}".format(algorithm,"top"+str(top_topic_words)) jgibbsLLDA(de_corpus, path2save_results=path2save_results, top_topic_words=top_topic_words) """ top_topic_words = 10 path2save_results = resultspath + "_{}_{}".format(algorithm,"top"+str(top_topic_words)) jgibbsLLDA(de_corpus, path2save_results=path2save_results, top_topic_words=top_topic_words) top_topic_words = 15 path2save_results = resultspath + "_{}_{}".format(algorithm, "top" + str(top_topic_words)) jgibbsLLDA(de_corpus, path2save_results=path2save_results, top_topic_words=top_topic_words) top_topic_words = 20 path2save_results = resultspath + "_{}_{}".format(algorithm, "top" + str(top_topic_words)) jgibbsLLDA(de_corpus, path2save_results=path2save_results, top_topic_words=top_topic_words) """ else: textacyTopicModeling(ngrams = 1, min_df = 1, max_df = 0.9, topicModel = algorithm, n_topics =15, corpus=de_corpus) """ textacyTopicModeling(ngrams=1, min_df=1, max_df=0.9, topicModel=algorithm, n_topics=20, corpus=de_corpus) textacyTopicModeling(ngrams=1, min_df=1, max_df=0.9, topicModel=algorithm, n_topics=25, corpus=de_corpus) textacyTopicModeling(ngrams=1, min_df=1, max_df=0.9, topicModel=algorithm, n_topics=30, corpus=de_corpus) """ textacyTopicModeling(ngrams=(1, 2), min_df=1, max_df=0.9, topicModel=algorithm, n_topics=15, corpus=de_corpus) """ textacyTopicModeling(ngrams = (1,2), min_df = 1, max_df = 0.9, topicModel = algorithm, n_topics =20, corpus=de_corpus) textacyTopicModeling(ngrams = (1,2), min_df = 1, max_df = 0.9, topicModel = algorithm, n_topics =25, corpus=de_corpus) textacyTopicModeling(ngrams = (1,2), min_df = 1, max_df = 0.9, topicModel = algorithm, n_topics =30, corpus=de_corpus) """ if __name__ == "__main__": main()