271 lines
7.4 KiB
Python
271 lines
7.4 KiB
Python
# -*- coding: utf-8 -*-
|
|
import csv
|
|
import functools
|
|
import re
|
|
import spacy
|
|
import sys
|
|
import textacy
|
|
import xml.etree.ElementTree as ET
|
|
import io
|
|
csv.field_size_limit(sys.maxsize)
|
|
|
|
|
|
|
|
# Load the configuration file
|
|
import configparser as ConfigParser
|
|
config = ConfigParser.ConfigParser()
|
|
with open("config.ini") as f:
|
|
config.read_file(f)
|
|
|
|
|
|
PARSER = spacy.load(config.get("default","language"))
|
|
corpus = textacy.Corpus(PARSER)
|
|
|
|
thesauruspath = config.get("default","thesauruspath")
|
|
THESAURUS = list(textacy.fileio.read_csv(thesauruspath, delimiter=";"))
|
|
|
|
|
|
|
|
def compose(*functions):
|
|
def compose2(f, g):
|
|
return lambda x: f(g(x))
|
|
return functools.reduce(compose2, functions, lambda x: x)
|
|
|
|
|
|
################ generate Content and Metadata ########################
|
|
|
|
def generateMainTextfromTicketXML(path2xml, main_textfield='Beschreibung'):
|
|
"""
|
|
generates strings from XML
|
|
:param path2xml:
|
|
:param main_textfield:
|
|
:param cleaning_function:
|
|
:yields strings
|
|
"""
|
|
tree = ET.parse(path2xml, ET.XMLParser(encoding="utf-8"))
|
|
root = tree.getroot()
|
|
|
|
for ticket in root:
|
|
for field in ticket:
|
|
if field.tag == main_textfield:
|
|
yield field.text
|
|
|
|
def generateMetadatafromTicketXML(path2xml, leave_out=['Beschreibung']):
|
|
tree = ET.parse(path2xml, ET.XMLParser(encoding="utf-8"))
|
|
root = tree.getroot()
|
|
|
|
for ticket in root:
|
|
metadata = {}
|
|
for field in ticket:
|
|
if field.tag not in leave_out:
|
|
|
|
metadata[field.tag] = field.text
|
|
|
|
yield metadata
|
|
|
|
def printRandomDoc(textacyCorpus):
|
|
import random
|
|
print()
|
|
|
|
print("len(textacyCorpus) = %i" % len(textacyCorpus))
|
|
randIndex = int((len(textacyCorpus) - 1) * random.random())
|
|
print("Index: {0} ; Text: {1} ; Metadata: {2}".format(randIndex, textacyCorpus[randIndex].text, textacyCorpus[randIndex].metadata))
|
|
|
|
print()
|
|
|
|
################ Preprocess#########################
|
|
|
|
def processDictstream(dictstream, funcdict, parser=PARSER):
|
|
for dic in dictstream:
|
|
result = {}
|
|
for key, value in dic.items():
|
|
if key in funcdict:
|
|
result[key] = funcdict[key](parser(value))
|
|
else:
|
|
result[key] = key
|
|
yield result
|
|
|
|
def processTextstream(textstream, func, parser=PARSER):
|
|
# input str-stream output str-stream
|
|
pipe = parser.pipe(textstream)
|
|
|
|
for doc in pipe:
|
|
yield func(doc)
|
|
|
|
|
|
|
|
|
|
def keepOnlyPOS(pos_list, parser=PARSER):
|
|
return lambda doc : parser(" ".join([tok.text for tok in doc if tok.pos_ in pos_list]))
|
|
|
|
def removeAllPOS(pos_list, parser=PARSER):
|
|
return lambda doc: parser(" ".join([tok.text for tok in doc if tok.pos_ not in pos_list]))
|
|
|
|
def keepOnlyENT(ent_list,parser=PARSER):
|
|
return lambda doc: parser(" ".join([tok.text for tok in doc if tok.ent_type_ in ent_list]))
|
|
|
|
def removeAllENT(ent_list, parser=PARSER):
|
|
return lambda doc: parser(" ".join([tok.text for tok in doc if tok.ent_type_ not in ent_list]))
|
|
|
|
|
|
|
|
|
|
doc2Set = lambda doc: str(set([tok.text for tok in doc]))
|
|
doc2String = lambda doc : doc.text
|
|
|
|
|
|
mentionFinder = re.compile(r"@[a-z0-9_]{1,15}", re.IGNORECASE)
|
|
emailFinder = re.compile(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.IGNORECASE)
|
|
urlFinder = re.compile(r"^(?:https?:\/\/)?(?:www\.)?[a-zA-Z0-9./]+$", re.IGNORECASE)
|
|
|
|
def replaceURLs(replace_with="URL",parser=PARSER):
|
|
#return lambda doc: parser(textacy.preprocess.replace_urls(doc.text,replace_with=replace_with))
|
|
return lambda doc: parser(urlFinder.sub(replace_with,doc.text))
|
|
|
|
def replaceEmails(replace_with="EMAIL",parser=PARSER):
|
|
#return lambda doc: parser(textacy.preprocess.replace_emails(doc.text,replace_with=replace_with))
|
|
return lambda doc : parser(emailFinder.sub(replace_with, doc.text))
|
|
|
|
def replaceTwitterMentions(replace_with="TWITTER_MENTION",parser=PARSER):
|
|
return lambda doc : parser(mentionFinder.sub(replace_with, doc.text))
|
|
|
|
def replaceNumbers(replace_with="NUMBER",parser=PARSER):
|
|
return lambda doc: parser(textacy.preprocess.replace_numbers(doc.text, replace_with=replace_with))
|
|
|
|
def replacePhonenumbers(replace_with="PHONE",parser=PARSER):
|
|
return lambda doc: parser(textacy.preprocess.replace_phone_numbers(doc.text, replace_with=replace_with))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def resolveAbbreviations(parser=PARSER):
|
|
pass #todo
|
|
|
|
|
|
def removeWords(words, keep=None,parser=PARSER):
|
|
if hasattr(keep, '__iter__'):
|
|
for k in keep:
|
|
try:
|
|
words.remove(k)
|
|
except ValueError:
|
|
pass
|
|
return lambda doc : parser(" ".join([tok.text for tok in doc if tok.lower_ not in words]))
|
|
|
|
|
|
|
|
|
|
def normalizeSynonyms(default_return_first_Syn=False, parser=PARSER):
|
|
#return lambda doc : parser(" ".join([tok.lower_ for tok in doc]))
|
|
return lambda doc : parser(" ".join([getFirstSynonym(tok.lower_, THESAURUS, default_return_first_Syn=default_return_first_Syn) for tok in doc]))
|
|
|
|
def getFirstSynonym(word, thesaurus, default_return_first_Syn=False):
|
|
if not isinstance(word, str):
|
|
return str(word)
|
|
|
|
word = word.lower()
|
|
|
|
# durch den thesaurrus iterieren
|
|
for syn_block in thesaurus: # syn_block ist eine liste mit Synonymen
|
|
|
|
for syn in syn_block:
|
|
syn = syn.lower()
|
|
if re.match(r'\A[\w-]+\Z', syn): # falls syn einzelwort ist
|
|
if word == syn:
|
|
return str(getHauptform(syn_block, word, default_return_first_Syn=default_return_first_Syn))
|
|
else: # falls es ein satz ist
|
|
if word in syn:
|
|
return str(getHauptform(syn_block, word, default_return_first_Syn=default_return_first_Syn))
|
|
return str(word) # zur Not, das ursrpüngliche Wort zurückgeben
|
|
|
|
def getHauptform(syn_block, word, default_return_first_Syn=False):
|
|
for syn in syn_block:
|
|
syn = syn.lower()
|
|
|
|
if "hauptform" in syn and len(syn.split(" ")) <= 2:
|
|
# nicht ausgeben, falls es in Klammern steht#todo gibts macnmal?? klammern aus
|
|
for w in syn.split(" "):
|
|
if not re.match(r'\([^)]+\)', w):
|
|
return w
|
|
|
|
if default_return_first_Syn:
|
|
# falls keine hauptform enthalten ist, das erste Synonym zurückgeben, was kein satz ist und nicht in klammern steht
|
|
for w in syn_block:
|
|
if not re.match(r'\([^)]+\)', w):
|
|
return w
|
|
return word # zur Not, das ursrpüngliche Wort zurückgeben
|
|
|
|
|
|
|
|
stop_words=list(__import__("spacy." + PARSER.lang, globals(), locals(), ['object']).STOP_WORDS) + config.get("preprocessing","custom_words").split(",")
|
|
|
|
path2xml = config.get("default","path2xml")
|
|
|
|
|
|
|
|
content_generator = generateMainTextfromTicketXML(path2xml)
|
|
metadata_generator = generateMetadatafromTicketXML(path2xml)
|
|
|
|
|
|
|
|
ents = config.get("preprocessing","ents").split(",")
|
|
|
|
|
|
|
|
clean_in_content=compose(
|
|
|
|
doc2String,
|
|
#normalizeSynonyms(default_return_first_Syn=config.get("preprocessing","default_return_first_Syn")),
|
|
replaceEmails(),
|
|
replaceURLs(),
|
|
replaceTwitterMentions(),
|
|
removeWords(stop_words),
|
|
#removeAllPOS(["SPACE","PUNCT"]),
|
|
#removeAllENT(ents),
|
|
keepOnlyPOS(['NOUN'])
|
|
)
|
|
|
|
clean_in_meta = {
|
|
"Loesung":removeAllPOS(["SPACE"]),
|
|
"Zusammenfassung":removeAllPOS(["SPACE","PUNCT"])
|
|
}
|
|
|
|
|
|
contentStream = processTextstream(content_generator, func=clean_in_content)
|
|
metaStream = processDictstream(metadata_generator, funcdict=clean_in_meta)
|
|
|
|
|
|
corpus.add_texts(contentStream,metaStream)
|
|
print(corpus[0].text)
|
|
printRandomDoc(corpus)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|