123 lines
2.5 KiB
Python
123 lines
2.5 KiB
Python
|
# -*- coding: utf-8 -*-
|
||
|
|
import functools
|
||
|
|
import re
|
||
|
|
import xml.etree.ElementTree as ET
|
||
|
|
|
||
|
|
import spacy
|
||
|
|
import textacy
|
||
|
|
|
||
|
|
path2xml = "ticketSamples.xml"
|
||
|
|
import de_core_news_md
|
||
|
|
|
||
|
|
|
||
|
|
PARSER = de_core_news_md.load()
|
||
|
|
corpus = textacy.Corpus(PARSER)
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
def generateMainTextfromTicketXML(path2xml, main_textfield='Beschreibung'):
|
||
|
|
"""
|
||
|
|
generates strings from XML
|
||
|
|
:param path2xml:
|
||
|
|
:param main_textfield:
|
||
|
|
:param cleaning_function:
|
||
|
|
:yields strings
|
||
|
|
"""
|
||
|
|
tree = ET.parse(path2xml, ET.XMLParser(encoding="utf-8"))
|
||
|
|
root = tree.getroot()
|
||
|
|
|
||
|
|
for ticket in root:
|
||
|
|
for field in ticket:
|
||
|
|
if field.tag == main_textfield:
|
||
|
|
yield field.text
|
||
|
|
|
||
|
|
|
||
|
|
def printRandomDoc(textacyCorpus):
|
||
|
|
import random
|
||
|
|
print()
|
||
|
|
|
||
|
|
print("len(textacyCorpus) = %i" % len(textacyCorpus))
|
||
|
|
randIndex = int((len(textacyCorpus) - 1) * random.random())
|
||
|
|
print("Index: {0} ; Text: {1} ; Metadata: {2}".format(randIndex, textacyCorpus[randIndex].text, textacyCorpus[randIndex].metadata))
|
||
|
|
|
||
|
|
print()
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
def processTextstream(textstream, funclist, parser=PARSER):
|
||
|
|
# input:str-stream output:str-stream
|
||
|
|
pipe = parser.pipe(textstream)
|
||
|
|
|
||
|
|
for doc in pipe:
|
||
|
|
tokens = [tok for tok in doc]
|
||
|
|
for f in funclist:
|
||
|
|
tokens = filter(f,tokens)
|
||
|
|
#tokens = map(funclist,tokens)
|
||
|
|
yield " ".join([tok.lower_ for tok in tokens])
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
def keepPOS(pos_list):
|
||
|
|
return lambda tok : tok.pos_ in pos_list
|
||
|
|
|
||
|
|
def removePOS(pos_list):
|
||
|
|
return lambda tok : tok.pos_ not in pos_list
|
||
|
|
|
||
|
|
def removeWords(words, keep=None):
|
||
|
|
#todo in:str oder str-list
|
||
|
|
if hasattr(keep, '__iter__'):
|
||
|
|
for k in keep:
|
||
|
|
try:
|
||
|
|
words.remove(k)
|
||
|
|
except ValueError:
|
||
|
|
pass
|
||
|
|
return lambda tok : tok.lower_ not in words
|
||
|
|
|
||
|
|
emailFinder = re.compile(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.IGNORECASE)
|
||
|
|
|
||
|
|
|
||
|
|
def replaceEmails(replace_with="EMAIL"):
|
||
|
|
return lambda tok : emailFinder.sub(replace_with, tok.lower_)
|
||
|
|
|
||
|
|
|
||
|
|
stop_words=list(__import__("spacy." + PARSER.lang, globals(), locals(), ['object']).STOP_WORDS)
|
||
|
|
|
||
|
|
clean_in_content=[
|
||
|
|
removePOS(["SPACE"]),
|
||
|
|
removePOS(["PUNCT"]),
|
||
|
|
removeWords(stop_words,keep=["und"]),
|
||
|
|
replaceEmails
|
||
|
|
]
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
## add files to textacy-corpus,
|
||
|
|
print("add texts to textacy-corpus...")
|
||
|
|
corpus.add_texts(
|
||
|
|
processTextstream(generateMainTextfromTicketXML(path2xml), clean_in_content),
|
||
|
|
)
|
||
|
|
|
||
|
|
printRandomDoc(corpus)
|
||
|
|
|
||
|
|
#todo https://stackoverflow.com/questions/15200048/how-to-get-the-parameters-type-and-return-type-of-a-function
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|
||
|
|
|