# -*- coding: utf-8 -*- import matplotlib matplotlib.use('Agg') from datetime import datetime import draw import draw1 import time import numpy as np import operator import csv import sys import json import os.path import subprocess from textacy import Vectorizer, viz from miscellaneous import * import textacy from scipy import * import os csv.field_size_limit(sys.maxsize) FILEPATH = os.path.dirname(os.path.realpath(__file__)) + "/" # load config config_ini = FILEPATH + "config.ini" config = ConfigParser.ConfigParser() with open(config_ini) as f: config.read_file(f) def textacyTopicModeling(corpus, n_topics = 15, top_topic_words = 7, top_document_labels_per_topic = 5, ngrams = 1, min_df=1, max_df=0.9, topicModel='lda'): n_terms = int(n_topics * top_topic_words) sort_terms_by = 'seriation' # 'seriation', 'weight', 'index', 'alphabetical' rank_terms_by = 'corpus' # 'corpus', 'topic' logprint("#### Topic Modeling {0}".format(topicModel)) logprint(str("ngrams: {0}".format(ngrams))) logprint(str("min_df: {0}".format(min_df))) logprint(str("max_df: {0}".format(max_df))) logprint(str("n_topics: {0}".format(n_topics))) logprint("\n") start = time.time() # http://textacy.readthedocs.io/en/latest/api_reference.html#textacy.tm.topic_model.TopicModel.get_doc_topic_matrix weighting = ('tf' if topicModel == 'lda' else 'tfidf') ###### vectorize corpi vectorizer = Vectorizer(weighting=weighting, min_df=min_df, max_df=max_df) terms_list = (doc.to_terms_list(ngrams=ngrams, named_entities=False, as_strings=True) for doc in corpus) doc_term_matrix = vectorizer.fit_transform(terms_list) id2term = vectorizer.__getattribute__("id_to_term") ####### Initialize and train a topic model model = textacy.tm.TopicModel(topicModel, n_topics=n_topics) model.fit(doc_term_matrix) doc_topic_matrix = model.transform(doc_term_matrix) for topic_idx, top_terms in model.top_topic_terms(vectorizer.id_to_term, top_n=top_topic_words, weights=True): logprint('{0}: {1}'.format(topic_idx, str(top_terms))) for topic_idx, top_docs in model.top_topic_docs(doc_topic_matrix, top_n=top_document_labels_per_topic): logprint(topic_idx) for j in top_docs: logprint(corpus[j].metadata['categoryName']) ####################### termite plot ################################################################### grams_label = "uni" if ngrams == 1 else "bi" draw1.termite_plot(model,doc_term_matrix, vectorizer.id_to_term, n_terms=n_terms, sort_terms_by=sort_terms_by, rank_terms_by=rank_terms_by + '_weight', save=FILEPATH + "results/{}_{}_{}_{}_{}_{}.png".format(grams_label, topicModel, n_topics, n_terms, sort_terms_by, rank_terms_by)) end = time.time() logprint("\n\n\nTime Elapsed Topic Modeling with {1}:{0} min\n\n".format((end - start) / 60, topicModel)) def jgibbsLLDA(labeldict,line_gen,path2save_results, top_topic_words=7): #labeldict = {k : labelist.count(k) for k in labelist} #max=0 #for v in labeldict.values(): # max = v if v > max else max #labelist = sort_dictionary(labeldict) #labeldict.update({'DEFAULT' : max+1}) labeldict_rev = {v: k for k, v in labeldict.items()} jgibbsLLDA_root = FILEPATH + "java_LabledLDA/" LLDA_filepath = "{0}models/tickets/tickets.gz".format(jgibbsLLDA_root) textacy.fileio.write_file_lines(line_gen, filepath=LLDA_filepath) # wait for file to exist while not os.path.exists(LLDA_filepath): time.sleep(1) # run JGibbsLLDA file n_topics = len(labeldict) #+1 #default-topic FNULL = open(os.devnull, 'w') # supress output cmd_jgibbs_java = ["java", "-cp", "{0}lib/trove-3.0.3.jar:{0}lib/args4j-2.0.6.jar:{0}out/production/LabledLDA/".format( jgibbsLLDA_root), "jgibblda.LDA", "-est", "-dir", "{0}models/tickets".format(jgibbsLLDA_root), "-dfile", "tickets.gz", "-twords", str(top_topic_words), "-ntopics", str(n_topics)] subprocess.call(cmd_jgibbs_java, stdout=FNULL) # ANMERKUNG: Dateien sind versteckt. zu finden in models/ cmd_gzip = ["gzip", "-dc", "{0}/models/tickets/.twords.gz".format(jgibbsLLDA_root)] output = subprocess.check_output(cmd_gzip).decode("utf-8") topic_regex = re.compile(r'Topic [0-9]*') ##################################### # todo save results in file aufgrund von results result = [] for line in output.splitlines(): findall = topic_regex.findall(line) if len(findall) != 0: try: index = int(findall[0].split()[1]) result.append("Topic {} {}:".format(index, labeldict_rev[index])) except: result.append(line) else: result.append(line) textacy.fileio.write_file_lines(result, path2save_results+".txt") ##################################### results = [] res_dict = {} count =0 for line in output.splitlines(): findall = topic_regex.findall(line) if len(findall) != 0: if len(res_dict) != 0: results.append(res_dict) #vorheriges an die liste ran (ist ja dann fertig) index = int(findall[0].split()[1]) res_dict = {index : str(labeldict_rev[index]) } else: splitted = line.split() res_dict[splitted[0]] = float(splitted[1]) if len(res_dict) != 0: results.append(res_dict) # letzes an die liste ran # every term in the resulsts to a list terms=[] for res in results: for key,value in res.items(): if not isinstance(key, int) and not key in terms: terms.append(key) term2id = {t:i for i,t in enumerate(terms)} #and to dict ################# termite plot ##################################################################### topic_labels = list(range(len(labeldict))) term_labels = list(range(len(term2id))) #tuple([key for key in term2id.keys()]) term_topic_weights = np.zeros((len(term2id),len(topic_labels))) for i,res in enumerate(results): for key,value in res.items(): if not isinstance(key, int): term_topic_weights[term2id[key]][i] = value term_labels[term2id[key]] = key else: topic_labels[i] = labeldict_rev[key] draw.draw_termite( term_topic_weights, topic_labels, term_labels, save=path2save_results+".png") draw.draw_termite( term_topic_weights, topic_labels, term_labels, save=path2save_results+"_spaced.png",pow_x=0.78,pow_y=0.87) # save labeldict labeldict_path = path2save_results + "_labeldict.json" with open(labeldict_path, 'w') as file: file.write(json.dumps(labeldict)) def jgibbsLLDA_category(corpus, path2save_results, top_topic_words=7): start = time.time() logprint("") logprint("start Category-LLDA:") # build dictionary of ticketcategories labelist = [] for doc in corpus: category = normalize(doc.metadata["categoryName"]) labelist.append(category) # frage nur die x häufigsten labels benutzen, rest raus? labelist = [l for l in labelist if labelist.count(l) > 50 ] in_labelist_ = {k: labelist.count(k) for k in labelist} labelist = sort_dictionary(in_labelist_) labelist.reverse() labeldict = {elem[0] : i for i, elem in enumerate(labelist)} #for elem in labelist: # l = elem[0] # c = elem[1] #labeldict = {elem[0] : len(labelist)-(i+1) for i, elem in enumerate(labelist)} #labelist = list(set(labelist)) #labeldict = {k: v for v, k in enumerate(labelist)} labeldict.update({'DEFAULT': len(labelist)}) def gen_cat_lines(textacyCorpus, labeldict): """ generates [topic1, topic2....] tok1 tok2 tok3 out of corpi""" for doc in textacyCorpus: label = labeldict.get(normalize(doc.metadata["categoryName"]), labeldict['DEFAULT']) if label is not 'DEFAULT': yield "[ " + str(label) + " ] " + doc.text line_gen = gen_cat_lines(corpus, labeldict) path2save_results = path2save_results + "_kb_cat_llda_{}".format("top" + str(top_topic_words)) jgibbsLLDA(labeldict, line_gen, path2save_results, top_topic_words=top_topic_words) end = time.time() logprint("\n\n\nTime Elapsed Category-LLDA :{0} min\n\n".format((end - start) / 60)) @deprecated def jgibbsLLDA_KB(corpus, path2save_results, top_topic_words = 7, kb_keywords=False): """ticket_ID -> KB_ID -> keywords / subject -> llda""" start = time.time() logprint("") logprint("start {}-LLDA:".format("Keyword" if kb_keywords else "Subject")) # ticket2kb_dict kb2ticket_gen = textacy.fileio.read_csv(FILEPATH + "M42-Export/KB2Ticket_2017-09-13.csv", delimiter=";") ticket2kb_dict = {} for line in kb2ticket_gen: ticket_id = line[0] kb_id = line[1] ticket2kb_dict[ticket_id] = kb_id # {'INC55646': 'KBA10065', 'INC65776': 'KBA10040', 'INC43025': 'KBA10056', ...} kb_entries_used = len(list(set(ticket2kb_dict.values()))) print("kb_entries_used: {}".format(kb_entries_used)) # kb2keywords_dict kb2keywords_gen = textacy.fileio.read_csv(FILEPATH + "M42-Export/KB_2017-09-13.csv", delimiter=";") next(kb2keywords_gen,None) #skip first line("ArticleID";"Subject";"Keywords";...) kb2keywords_dict = {} for line in kb2keywords_gen: kb_id = line[0] subject = line[1] keywords = line[2] keywords_list = [normalize(x) for x in str(keywords).split(",")] if kb_id not in kb2keywords_dict.keys(): kb2keywords_dict[kb_id] = [] if kb_keywords: for item in keywords_list: if item != "": kb2keywords_dict[kb_id].append(item) else: kb2keywords_dict[kb_id].append(subject) #remove all empty items kb2keywords_dict = { k : v for k,v in kb2keywords_dict.items() if len(v) != 0} # {'KBA10091': ['citavi'], 'KBA10249': ['"beschaedigte unicard"', 'risse', '"defekte karte"'], ...} #keywords2kb_dict keywords2kb_dict = {} for kb_id, lst in kb2keywords_dict.items(): for l in lst: if l not in keywords2kb_dict.keys(): keywords2kb_dict[l] = [kb_id] else: keywords2kb_dict[l].append(kb_id) # {'unicard namensaenderung': ['KBA10276'], 'vpn': ['KBA10063'], 'outlook_exchange': ['KBA10181'], ...} # Look for actually used keywords used_keywords = [] for doc in corpus: ticket_number = doc.metadata["TicketNumber"] kb_id = ticket2kb_dict.get(ticket_number, None) keywords = kb2keywords_dict.get(kb_id, None) if keywords and kb_id: used_keywords.append(list(map(normalize,keywords))) labelist = [item for sublist in used_keywords for item in sublist] #flatten list labelist = list(set(labelist)) print("len(labelist): {}".format(len(labelist))) labeldict = {k: v for v, k in enumerate(labelist)} def gen_KB_lines(textacyCorpus, labeldict, ticket2kb_dict, kb2keywords_dict): for doc in corpus: ticket_number = doc.metadata["TicketNumber"] kb_number = ticket2kb_dict.get(ticket_number, None) keywords = kb2keywords_dict.get(kb_number, None) if keywords: label = "" for kw in keywords: label = label + str(labeldict.get(normalize(str(kw)), len(labeldict))) + " " yield "[ " + label + "] " + doc.text line_gen = gen_KB_lines(corpus, labeldict, ticket2kb_dict, kb2keywords_dict) path2save_results = path2save_results + "_kb_{}_llda_{}".format("keys" if kb_keywords else "subs", "top" + str(top_topic_words)) jgibbsLLDA(labeldict, line_gen, path2save_results, top_topic_words=top_topic_words) end = time.time() logprint("\n\n\nTime Elapsed {1}-LLDA :{0} min\n\n".format((end - start) / 60,"Keyword" if kb_keywords else "Subject")) def jgibbsLLDA_KB_v2(corpus, path2save_results, top_topic_words = 7): start = time.time() logprint("") logprint("start LLDA:") # kb2keywords_dict / kb2subjects_dict --> {str : [str]} kb2keywords_dict = {} kb2subjects_dict = {} kb_gen = textacy.fileio.read_csv(FILEPATH + "M42-Export/KB_2017-09-13.csv", delimiter=";") next(kb_gen, None) # skip first line "ArticleID";"Subject";"Keywords";... for line in kb_gen: kb_id = line[0] subject = normalize(line[1]) keywords = [normalize(x) for x in str(line[2]).split(",")] if kb_id not in kb2keywords_dict.keys(): kb2keywords_dict[kb_id] = keywords if keywords != [''] else ["DEFAULT"] else: kb2keywords_dict[kb_id] = kb2keywords_dict[kb_id] + keywords if kb_id not in kb2subjects_dict.keys(): kb2subjects_dict[kb_id] = [normalize(subject) if subject != [''] else "DEFAULT"] else: kb2subjects_dict[kb_id].append(normalize(subject)) # ticket2kbs_dict --> {str : [str]} ticket2kbs_dict = {} kb2ticket_gen = textacy.fileio.read_csv(FILEPATH + "M42-Export/KB2Ticket_2017-09-13.csv", delimiter=";") next(kb2ticket_gen, None) # skip first line"TicketNumber";"ArticleID" for line in kb2ticket_gen: ticket_id = line[0] kb_id = line[1] if ticket_id not in ticket2kbs_dict.keys(): ticket2kbs_dict[ticket_id] = [kb_id] else: ticket2kbs_dict[ticket_id].append(kb_id) # ticket2keywords --> {str:[str]} ticket2keywords_dict = {} for ticket_id, kb_ids in ticket2kbs_dict.items(): if ticket_id not in ticket2keywords_dict.keys(): ticket2keywords_dict[ticket_id] = [] for kb_id in kb_ids: ticket2keywords_dict[ticket_id].append(kb2keywords_dict[kb_id]) ticket2keywords_dict[ticket_id] = flatten(ticket2keywords_dict[ticket_id]) # ticket2subjects --> {str:[str]} ticket2subjects_dict = {} for ticket_id, kb_ids in ticket2kbs_dict.items(): if ticket_id not in ticket2subjects_dict.keys(): ticket2subjects_dict[ticket_id] = [] for kb_id in kb_ids: ticket2subjects_dict[ticket_id].append(kb2subjects_dict[kb_id]) ticket2subjects_dict[ticket_id] = flatten(ticket2subjects_dict[ticket_id]) # kb2keywords_dict {'KBA10230': ['DEFAULT'], 'KBA10129': ['DEFAULT'], 'KBA10287': ['sd_ansys_informationen'], } len = 260 # kb2subjects_dict {'KBA10230': ['unicard nochmal beantragen'], 'KBA10129': ['sd_entsperrung unicard nach verlust/wiederfinden'], } len = 260 # ticket2kbs_dict {'INC44526': ['KBA10056'], 'INC67205': ['KBA10056'], } len = 4832 # ticket2keywords_dict {'INC44526': ['DEFAULT'], 'INC67205': ['DEFAULT'], 'INC71863': ['DEFAULT'], 'INC44392': ['asknet'] } len=4832 # ticket2subjects_dict {'INC44526': ['sd_telefon (antrag: neuanschluss, umzug, aenderung erledigt)'], len=4832 count_dict = {} for v in ticket2kbs_dict.values(): for kb in v: if kb in count_dict.keys(): count_dict[kb] +=1 else: count_dict[kb] = 1 sorted_dict = sorted(count_dict.items(), key=operator.itemgetter(1)) """ for k,v in sorted_dict: subs = kb2subjects_dict[k] keys = kb2keywords_dict[k] print(subs, keys , v) # frage wieviele tickets pro topic? print("kb_entrys used: {}".format(len(sorted_dict))) # frage wie viele kb_entry's insg genutzt?: 155 """ labelist = ticket2keywords_dict.values() labelist = flatten(labelist) labelist = list(set(labelist)) labeldict = {k: v for v, k in enumerate(labelist)} def gen_key_lines(textacyCorpus, labeldict, ticket2keywords_dict): for doc in corpus: ticket_number = doc.metadata["TicketNumber"] keywords = ticket2keywords_dict.get(ticket_number, ['DEFAULT']) if keywords != ['DEFAULT']: label = "" for kw in keywords: label = label + str(labeldict.get(normalize(str(kw)), labeldict['DEFAULT'])) + " " yield "[ " + label + "] " + doc.text keys_line_gen = gen_key_lines(corpus, labeldict, ticket2keywords_dict) path2save_keys_results = path2save_results + "_kb_keys_llda_{}".format("top" + str(top_topic_words)) jgibbsLLDA(labeldict, keys_line_gen, path2save_keys_results, top_topic_words=top_topic_words) labelist = ticket2subjects_dict.values() labelist = flatten(labelist) labelist = list(set(labelist)) labeldict = {k: v for v, k in enumerate(labelist)} labeldict.update({'DEFAULT' : len(labeldict)}) subj_line_gen = gen_key_lines(corpus, labeldict, ticket2subjects_dict) path2save_subj_results = path2save_results + "_kb_subj_llda_{}".format("top" + str(top_topic_words)) jgibbsLLDA(labeldict, subj_line_gen, path2save_subj_results, top_topic_words=top_topic_words) end = time.time() logprint("\n\n\nTime Elapsed LLDA :{0} min\n\n".format((end - start) / 60)) def load_from_labled_lines(path): path = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/corpi/pre_labled_lines_wo_lemma_061217.txt" #idee plan # clean laden, pre laden # unigramme und num/wort-bigramme doc-term # frage wie geht llda mit bigrammen um? idee bigramme mit _ verbinden # nimm nur ngrams wo midn. ein token in pre vorkommt def main(cleaned_corpus, pre_corpus, algorithm="llda"): logprint("Topic Modeling: {0}".format(datetime.now())) #todo von labled_lines laden ?? #idee thesaurus vor id2term #todo akronyme & abk. drin lassen #todo bigramme nicht auf pre, sondern auf cleaned #todo zahlen drin lassen, bigramme: NUM wort kombis #todo levenstein/hamming distanz statt autokorrekt #idee oder word2vec #todo ticket-subj mit einbeziehen resultspath = FILEPATH + "results/pre" de_corpus = pre_corpus if algorithm == "llda": top_topic_words = 3 jgibbsLLDA_category(de_corpus, path2save_results=resultspath, top_topic_words=top_topic_words) jgibbsLLDA_KB_v2(de_corpus, path2save_results=resultspath, top_topic_words=top_topic_words) """ kb_keywords = False jgibbsLLDA_KB(de_corpus, path2save_results=resultspath, top_topic_words=top_topic_words, kb_keywords=kb_keywords) kb_keywords = True jgibbsLLDA_KB(de_corpus, path2save_results=resultspath, top_topic_words=top_topic_words, kb_keywords=kb_keywords) top_topic_words = 10 path2save_results = resultspath + "_{}_{}".format(algorithm,"top"+str(top_topic_words)) jgibbsLLDA(de_corpus, path2save_results=path2save_results, top_topic_words=top_topic_words) top_topic_words = 15 path2save_results = resultspath + "_{}_{}".format(algorithm, "top" + str(top_topic_words)) jgibbsLLDA(de_corpus, path2save_results=path2save_results, top_topic_words=top_topic_words) top_topic_words = 20 path2save_results = resultspath + "_{}_{}".format(algorithm, "top" + str(top_topic_words)) jgibbsLLDA(de_corpus, path2save_results=path2save_results, top_topic_words=top_topic_words) """ else: textacyTopicModeling(ngrams = 1, topicModel = algorithm, corpus=de_corpus) """ textacyTopicModeling(ngrams=1, min_df=1, max_df=0.9, topicModel=algorithm, n_topics=20, corpus=de_corpus) textacyTopicModeling(ngrams=1, min_df=1, max_df=0.9, topicModel=algorithm, n_topics=25, corpus=de_corpus) textacyTopicModeling(ngrams=1, min_df=1, max_df=0.9, topicModel=algorithm, n_topics=30, corpus=de_corpus) """ textacyTopicModeling(ngrams=(1, 2), topicModel=algorithm, corpus=de_corpus) """ textacyTopicModeling(ngrams = (1,2), min_df = 1, max_df = 0.9, topicModel = algorithm, n_topics =20, corpus=de_corpus) textacyTopicModeling(ngrams = (1,2), min_df = 1, max_df = 0.9, topicModel = algorithm, n_topics =25, corpus=de_corpus) textacyTopicModeling(ngrams = (1,2), min_df = 1, max_df = 0.9, topicModel = algorithm, n_topics =30, corpus=de_corpus) """ if __name__ == "__main__": # load corpus corpus_de_path = FILEPATH + config.get("de_corpus", "path") pre_corpus_name = "de" + "_pre" pre_corpus, parser = load_corpus(corpus_name=pre_corpus_name, corpus_path=corpus_de_path) logprint("Corpus loaded: {0}".format(pre_corpus_name)) cleaned_corpus_name = "de" + "_raw" #cleaned_corpus, parser = load_corpus(corpus_name=cleaned_corpus_name, corpus_path=corpus_de_path) logprint("Corpus loaded: {0}".format(cleaned_corpus_name)) cleaned_corpus = None main(pre_corpus=pre_corpus,cleaned_corpus=cleaned_corpus,algorithm="llda") main(pre_corpus=pre_corpus,cleaned_corpus=cleaned_corpus,algorithm="lda")