349 lines
9.7 KiB
Python
349 lines
9.7 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
|
|
from datetime import datetime
|
|
|
|
print(datetime.now())
|
|
|
|
import time
|
|
|
|
import enchant
|
|
|
|
start = time.time()
|
|
|
|
import logging
|
|
|
|
import csv
|
|
import functools
|
|
import os.path
|
|
import re
|
|
import subprocess
|
|
import time
|
|
import xml.etree.ElementTree as ET
|
|
import sys
|
|
import spacy
|
|
import textacy
|
|
from scipy import *
|
|
from textacy import Vectorizer
|
|
import warnings
|
|
import configparser as ConfigParser
|
|
import sys
|
|
import hunspell
|
|
from postal.parser import parse_address
|
|
|
|
csv.field_size_limit(sys.maxsize)
|
|
|
|
|
|
def printlog(string, level="INFO"):
|
|
"""log and prints"""
|
|
print(string)
|
|
if level == "INFO":
|
|
logging.info(string)
|
|
elif level == "DEBUG":
|
|
logging.debug(string)
|
|
elif level == "WARNING":
|
|
logging.warning(string)
|
|
|
|
|
|
printlog("Load functions")
|
|
|
|
def printRandomDoc(textacyCorpus):
|
|
import random
|
|
print()
|
|
|
|
printlog("len(textacyCorpus) = %i" % len(textacyCorpus))
|
|
randIndex = int((len(textacyCorpus) - 1) * random.random())
|
|
printlog("Index: {0} ; Text: {1} ; Metadata: {2}\n".format(randIndex, textacyCorpus[randIndex].text,
|
|
textacyCorpus[randIndex].metadata))
|
|
|
|
print()
|
|
|
|
|
|
def load_corpus(corpus_path,corpus_name):
|
|
# load new lang
|
|
nlp = spacy.load("de")
|
|
|
|
#load stringstore
|
|
stringstore_path = corpus_path + corpus_name + '_strings.json'
|
|
with open(stringstore_path,"r") as file:
|
|
nlp.vocab.strings.load(file)
|
|
|
|
# define corpus
|
|
corpus = textacy.Corpus(nlp)
|
|
|
|
# load meta
|
|
metapath = corpus_path + corpus_name +"_meta.json"
|
|
metadata_stream = textacy.fileio.read_json_lines(metapath)
|
|
|
|
#load content
|
|
contentpath = corpus_path + corpus_name+ "_content.bin"
|
|
spacy_docs = textacy.fileio.read_spacy_docs(corpus.spacy_vocab, contentpath)
|
|
|
|
for spacy_doc, metadata in zip(spacy_docs, metadata_stream):
|
|
corpus.add_doc(
|
|
textacy.Doc(spacy_doc, lang=corpus.spacy_lang, metadata=metadata))
|
|
|
|
return corpus
|
|
|
|
|
|
def printvecotorization(ngrams=1, min_df=1, max_df=1.0, weighting='tf', named_entities=True):
|
|
printlog(str("ngrams: {0}".format(ngrams)))
|
|
printlog(str("min_df: {0}".format(min_df)))
|
|
printlog(str("max_df: {0}".format(max_df)))
|
|
printlog(str("named_entities: {0}".format(named_entities)))
|
|
|
|
# printlog("vectorize corpus...")
|
|
vectorizer = Vectorizer(weighting=weighting, min_df=min_df, max_df=max_df)
|
|
|
|
terms_list = (doc.to_terms_list(ngrams=ngrams, named_entities=named_entities, as_strings=True) for doc in de_corpus)
|
|
doc_term_matrix = vectorizer.fit_transform(terms_list)
|
|
id2term = vectorizer.__getattribute__("id_to_term")
|
|
|
|
for t in terms_list:
|
|
print(t)
|
|
printlog("doc_term_matrix: {0}".format(doc_term_matrix))
|
|
printlog("id2term: {0}".format(id2term))
|
|
|
|
|
|
|
|
|
|
corpus_path = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/corpus/"
|
|
corpus_name = "de_corpus"
|
|
|
|
# load corpus
|
|
de_corpus = load_corpus(corpus_name=corpus_name,corpus_path=corpus_path)
|
|
|
|
|
|
for i in range(5):
|
|
printRandomDoc(de_corpus)
|
|
|
|
|
|
|
|
|
|
# todo gescheites tf(-idf) maß finden
|
|
ngrams = 1
|
|
min_df = 1
|
|
max_df = 1.0
|
|
weighting = 'tf'
|
|
# weighting ='tfidf'
|
|
named_entities = False
|
|
|
|
"""
|
|
printvecotorization(ngrams=1, min_df=1, max_df=1.0, weighting=weighting)
|
|
printvecotorization(ngrams=1, min_df=1, max_df=0.5, weighting=weighting)
|
|
printvecotorization(ngrams=1, min_df=1, max_df=0.8, weighting=weighting)
|
|
|
|
printvecotorization(ngrams=(1, 2), min_df=1, max_df=1.0, weighting=weighting)
|
|
printvecotorization(ngrams=(1, 2), min_df=1, max_df=0.5, weighting=weighting)
|
|
printvecotorization(ngrams=(1, 2), min_df=1, max_df=0.8, weighting=weighting)
|
|
"""
|
|
|
|
# build citionary of ticketcategories
|
|
labelist = []
|
|
|
|
for texdoc in de_corpus.get(lambda texdoc: texdoc.metadata["categoryName"] not in labelist):
|
|
labelist.append(texdoc.metadata["categoryName"])
|
|
|
|
LABELDICT = {k: v for v, k in enumerate(labelist)}
|
|
|
|
printlog(str("LABELDICT: {0}".format(LABELDICT)))
|
|
|
|
|
|
def textacyTopicModeling(ngrams, min_df, max_df, topicModel='lda', n_topics=len(LABELDICT), named_entities=False,
|
|
corpus=de_corpus):
|
|
printlog(
|
|
"############################################ Topic Modeling {0} #############################################".format(
|
|
topicModel))
|
|
print("\n\n")
|
|
printlog(str("ngrams: {0}".format(ngrams)))
|
|
printlog(str("min_df: {0}".format(min_df)))
|
|
printlog(str("max_df: {0}".format(max_df)))
|
|
printlog(str("n_topics: {0}".format(n_topics)))
|
|
printlog(str("named_entities: {0}".format(named_entities)))
|
|
|
|
start = time.time()
|
|
|
|
top_topic_words = 10
|
|
top_document_labels_per_topic = 5
|
|
|
|
# http://textacy.readthedocs.io/en/latest/api_reference.html#textacy.tm.topic_model.TopicModel.get_doc_topic_matrix
|
|
weighting = ('tf' if topicModel == 'lda' else 'tfidf')
|
|
|
|
####################'####################
|
|
|
|
|
|
# printlog("vectorize corpus...")
|
|
vectorizer = Vectorizer(weighting=weighting, min_df=min_df, max_df=max_df)
|
|
|
|
terms_list = (doc.to_terms_list(ngrams=ngrams, named_entities=named_entities, as_strings=True) for doc in corpus)
|
|
doc_term_matrix = vectorizer.fit_transform(terms_list)
|
|
id2term = vectorizer.__getattribute__("id_to_term")
|
|
|
|
# printlog("terms_list: {0}".format(list(terms_list)))
|
|
# printlog("doc_term_matrix: {0}".format(doc_term_matrix))
|
|
|
|
|
|
|
|
##################### LSA, LDA, NMF Topic Modeling via Textacy ##############################################
|
|
|
|
# Initialize and train a topic model
|
|
# printlog("Initialize and train a topic model..")
|
|
model = textacy.tm.TopicModel(topicModel, n_topics=n_topics)
|
|
model.fit(doc_term_matrix)
|
|
|
|
# Transform the corpus and interpret our model:
|
|
# printlog("Transform the corpus and interpret our model..")
|
|
doc_topic_matrix = model.transform(doc_term_matrix)
|
|
print()
|
|
|
|
for topic_idx, top_terms in model.top_topic_terms(vectorizer.id_to_term, top_n=top_topic_words):
|
|
printlog('topic {0}: {1}'.format(topic_idx, " ".join(top_terms)))
|
|
|
|
print()
|
|
for topic_idx, top_docs in model.top_topic_docs(doc_topic_matrix, top_n=top_document_labels_per_topic):
|
|
printlog(topic_idx)
|
|
for j in top_docs:
|
|
printlog(corpus[j].metadata['categoryName'])
|
|
print()
|
|
|
|
#####################################################################################################################
|
|
print()
|
|
print()
|
|
|
|
end = time.time()
|
|
printlog("\n\n\nTime Elapsed Topic Modeling with {1}:{0} min\n\n".format((end - start) / 60, topicModel))
|
|
|
|
|
|
# no_below = 20
|
|
# no_above = 0.5
|
|
|
|
|
|
# n_topics = len(LABELDICT)#len(set(ticketcorpus[0].metadata.keys()))+1 #+1 wegen einem default-topic
|
|
|
|
|
|
|
|
"""
|
|
topicModeling(ngrams = 1,
|
|
min_df = 1,
|
|
max_df = 1.0,
|
|
topicModel = 'lda',
|
|
n_topics = len(LABELDICT),
|
|
corpus=de_corpus)
|
|
|
|
topicModeling(ngrams = 1,
|
|
min_df = 0.1,
|
|
max_df = 0.6,
|
|
topicModel = 'lda',
|
|
n_topics = len(LABELDICT),
|
|
corpus=de_corpus)
|
|
|
|
topicModeling(ngrams = (1,2),
|
|
min_df = 1,
|
|
max_df = 1.0,
|
|
topicModel = 'lda',
|
|
n_topics = len(LABELDICT),
|
|
corpus=de_corpus)
|
|
|
|
topicModeling(ngrams = (1,2),
|
|
min_df = 0.1,
|
|
max_df = 0.6,
|
|
topicModel = 'lda',
|
|
n_topics = len(LABELDICT),
|
|
corpus=de_corpus)
|
|
|
|
topicModeling(ngrams = (1,2),
|
|
min_df = 0.2,
|
|
max_df = 0.8,
|
|
topicModel = 'lda',
|
|
n_topics = 20,
|
|
corpus=de_corpus)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
##################### LLDA Topic Modeling via JGibbsLabledLDA ##############################################
|
|
|
|
|
|
top_topic_words = 15
|
|
|
|
print("\n\n")
|
|
start = time.time()
|
|
|
|
n_topics = len(LABELDICT) # len(set(ticketcorpus[0].metadata.keys()))+1 #+1 wegen einem default-topic
|
|
|
|
# build citionary of ticketcategories
|
|
labelist = []
|
|
|
|
for texdoc in de_corpus.get(lambda texdoc: texdoc.metadata["categoryName"] not in labelist):
|
|
labelist.append(texdoc.metadata["categoryName"])
|
|
|
|
LABELDICT = {k: v for v, k in enumerate(labelist)}
|
|
print(LABELDICT)
|
|
|
|
|
|
def label2ID(label, labeldict=LABELDICT):
|
|
return labeldict.get(label, len(labeldict))
|
|
|
|
|
|
def generate_labled_lines(textacyCorpus):
|
|
for doc in textacyCorpus:
|
|
# generate [topic1, topic2....] tok1 tok2 tok3 out of corpus
|
|
yield "[" + str(label2ID(doc.metadata["categoryName"])) + "] " + doc.text
|
|
|
|
|
|
jgibbsLLDA_root = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/java_LabledLDA/"
|
|
LLDA_filepath = "{0}models/tickets/tickets.gz".format(jgibbsLLDA_root)
|
|
|
|
# create file
|
|
textacy.fileio.write_file_lines(generate_labled_lines(de_corpus), filepath=LLDA_filepath)
|
|
|
|
# todfo ticket drucken
|
|
# wait for file to exist
|
|
while not os.path.exists(LLDA_filepath):
|
|
time.sleep(1)
|
|
|
|
print("\n\n")
|
|
printlog("start LLDA:")
|
|
# run JGibsslda file
|
|
FNULL = open(os.devnull, 'w') # supress output
|
|
subprocess.call(["java",
|
|
"-cp",
|
|
"{0}lib/trove-3.0.3.jar:{0}lib/args4j-2.0.6.jar:{0}out/production/LabledLDA/".format(jgibbsLLDA_root),
|
|
"jgibblda.LDA",
|
|
"-est",
|
|
"-dir", "{0}models/tickets".format(jgibbsLLDA_root),
|
|
"-dfile", "tickets.gz",
|
|
"-twords", str(top_topic_words),
|
|
"-ntopics", str(n_topics)], stdout=FNULL)
|
|
|
|
# ANMERKUNG: Dateien sind versteckt. zu finden in models/
|
|
|
|
# twords
|
|
subprocess.call(["gzip",
|
|
"-dc",
|
|
"{0}/models/tickets/.twords.gz".format(jgibbsLLDA_root)])
|
|
#####################################################################################################################
|
|
print()
|
|
print()
|
|
|
|
end = time.time()
|
|
printlog("\n\n\nTime Elapsed Topic Modeling JGibbsLLDA:{0} min\n\n".format((end - start) / 60))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|