# -*- coding: utf-8 -*- import re import time import spacy import textacy start = time.time() from datetime import datetime import xml.etree.ElementTree as ET print(datetime.now()) nomen=[] #PARSER=spacy.load("de") #todo: thesaurus....yay... """ def normalizeSynonyms(default_return_first_Syn=False, parser=PARSER): #return lambda doc : parser(" ".join([tok.lower_ for tok in doc])) return lambda doc : parser(" ".join([getFirstSynonym(tok.lower_, THESAURUS, default_return_first_Syn=default_return_first_Syn) for tok in doc])) def getFirstSynonym(word, thesaurus, default_return_first_Syn=False): if not isinstance(word, str): return str(word) word = word.lower() # durch den thesaurrus iterieren for syn_block in thesaurus: # syn_block ist eine liste mit Synonymen for syn in syn_block: syn = syn.lower() if re.match(r'\A[\w-]+\Z', syn): # falls syn einzelwort ist if word == syn: return str(getHauptform(syn_block, word, default_return_first_Syn=default_return_first_Syn)) else: # falls es ein satz ist if word in syn: return str(getHauptform(syn_block, word, default_return_first_Syn=default_return_first_Syn)) return str(word) # zur Not, das ursrpüngliche Wort zurückgeben def getHauptform(syn_block, word, default_return_first_Syn=False): for syn in syn_block: syn = syn.lower() if "hauptform" in syn and len(syn.split(" ")) <= 2: # nicht ausgeben, falls es in Klammern steht#todo gibts macnmal?? klammern aus for w in syn.split(" "): if not re.match(r'\([^)]+\)', w): return w if default_return_first_Syn: # falls keine hauptform enthalten ist, das erste Synonym zurückgeben, was kein satz ist und nicht in klammern steht for w in syn_block: if not re.match(r'\([^)]+\)', w): return w return word # zur Not, das ursrpüngliche Wort zurückgeben """ ### extract from deWordNet.xml #https://github.com/hdaSprachtechnologie/odenet #idee synsets bilden """ path2xml="/home/jannis.grundmann/PycharmProjects/topicModelingTickets/deWordNet.xml" tree = ET.parse(path2xml, ET.XMLParser(encoding="utf-8")) root = tree.getroot() for r in root: for element in r: if element.tag == "Synset": attrib = element.attrib for i,subentry in enumerate(element): if subentry.tag == "Lemma" and subentry.attrib["partOfSpeech"] == "n": string = (subentry.attrib["writtenForm"]) # replaceRockDots string = re.sub(r'[ß]', "ss", string) string = re.sub(r'[ö]', "oe", string) string = re.sub(r'[ü]', "ue", string) string = re.sub(r'[ä]', "ae", string) # seperate_words_on_regex: string = " ".join(re.compile(regex_specialChars).split(string)) string_list=string.split() if len(string_list) == 1: nomen.append(string.lower().strip()) """ lexicalentries = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/lexicalentries.xml" synsets = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/synsets.xml" lextree = ET.parse(lexicalentries, ET.XMLParser(encoding="utf-8")) syntree = ET.parse(synsets, ET.XMLParser(encoding="utf-8")) lexroot = lextree.getroot() synroot = syntree.getroot() for r in synroot: for element in r: if element.tag == "Synset": sysnet = [] attrib = element.attrib id = attrib["id"] for ro in lexroot: for elem in ro: if elem.tag == "LexicalEntry": subs_dicts = [subentry.attrib for subentry in elem] #: [{'partOfSpeech': 'n', 'writtenForm': 'Kernspaltung'}, {'synset': 'de-1-n', 'id': 'w1_1-n'}] dic = {k:v for x in subs_dicts for k,v in x.items()} # to one dict if "synset" in dic.keys(): if dic["synset"] == id: if id == "de-1004-n": x = 0 string = (dic["writtenForm"]) # replaceRockDots string = re.sub(r'[ß]', "ss", string) string = re.sub(r'[ö]', "oe", string) string = re.sub(r'[ü]', "ue", string) string = re.sub(r'[ä]', "ae", string) # alle punkte raus string = re.sub(r'[.]', "", string) # alles in klammern raus stringlist = string.split() strings=[] for w in stringlist: if not bool(re.match(r'/\(([^)]+)\)/', w)): #todo funzt nich wie's soll strings.append(w) string = " ".join(strings) #re.sub(r'/\(([^)]+)\)/', " ", string) sysnet.append(string.lower().strip()) print(id,sysnet) """ ### extract from derewo #http://www1.ids-mannheim.de/kl/projekte/methoden/derewo.html raw = textacy.fileio.read_file_lines("DeReKo-2014-II-MainArchive-STT.100000.freq") for line in raw: line_list=line.split() if line_list[2] == "NN": string = line_list[1].lower() # replaceRockDots string = re.sub(r'[ß]', "ss", string) string = re.sub(r'[ö]', "oe", string) string = re.sub(r'[ü]', "ue", string) string = re.sub(r'[ä]', "ae", string) nomen.append(string.lower().strip()) textacy.fileio.write_file_lines(nomen,"nomen2.txt") """ """ stream = textacy.fileio.read_csv("/home/jannis.grundmann/PycharmProjects/topicModelingTickets/M42-Export/Tickets_2017-09-13.csv", delimiter=";") content_collumn_name = "Description" content_collumn = 9 # standardvalue de_tickets=[] en_tickets=[] misc_tickets=[] error_count = 0 for i, lst in enumerate(stream): if i == 0: de_tickets.append(lst) en_tickets.append(lst) misc_tickets.append(lst) else: try: content_collumn_ = lst[content_collumn] if detect(content_collumn_) == "de": de_tickets.append(lst) elif detect(content_collumn_) == "en": en_tickets.append(lst) else: misc_tickets.append(lst) except: misc_tickets.append(lst) error_count += 1 print(error_count) textacy.fileio.write_csv(de_tickets,"M42-Export/de_tickets.csv", delimiter=";") textacy.fileio.write_csv(en_tickets,"M42-Export/en_tickets.csv", delimiter=";") textacy.fileio.write_csv(misc_tickets,"M42-Export/misc_tickets.csv", delimiter=";") """ """ regex_specialChars = r'[`\-=~!#@,.$%^&*()_+\[\]{};\'\\:"|?]' def stringcleaning(stringstream, funclist): for string in stringstream: for f in funclist: string = f(string) yield string def seperate_words_on_regex(regex=regex_specialChars): return lambda string: " ".join(re.compile(regex).split(string)) words = [ "uniaccount", "nr54065467", "nr54065467", "455a33c5," "tvt?=", "tanja.saborowski@tu-dortmund.de", "-", "m-sw1-vl4053.itmc.tu-dortmund.de", "------problem--------" ] topLVLFinder = re.compile(r'\.[a-z]{2,3}(\.[a-z]{2,3})?', re.IGNORECASE) specialFinder = re.compile(r'[`\-=~!@#$%^&*()_+\[\]{};\'\\:"|<,./>?]', re.IGNORECASE) for s in stringcleaning((w for w in words),[seperate_words_on_regex()]): print(s.strip()) #print(stringcleaning(w,string_comp)) #print(bool(re.search(r'\.[a-z]{2,3}(\.[a-z]{2,3})?',w))) #print(bool(re.search(r'[`\-=~!@#$%^&*()_+\[\]{};\'\\:"|<,./>?]',w))) #result = specialFinder.sub(" ", w) #print(re.sub(r'[`\-=~!@#$%^&*()_+\[\]{};\'\\:"|<,./>?]'," ",w)) #print(re.sub(r'\.[a-z]{2,3}(\.[a-z]{2,3})?', " ", w)) """ """ def replaceRockDots(): return lambda string: re.sub(r'[ß]', "ss", (re.sub(r'[ö]', "oe", (re.sub(r'[ü]', "ue", (re.sub(r'[ä]', "ae", string.lower()))))))) de_stop_words = list(textacy.fileio.read_file_lines(filepath="german_stopwords_full.txt")) #blob = Text(str(textacy.fileio.read_file("teststring.txt")))#,parser=PatternParser(pprint=True, lemmata=True)) #print(blob.entities) de_stop_words = list(map(replaceRockDots(),de_stop_words)) #LEMMAS = list(map(replaceRockDots(),LEMMAS)) #VORNAMEN = list(map(replaceRockDots(),VORNAMEN)) de_stop_words = list(map(textacy.preprocess.normalize_whitespace,de_stop_words)) #LEMMAS = list(map(textacy.preprocess.normalize_whitespace,LEMMAS)) #VORNAMEN = list(map(textacy.preprocess.normalize_whitespace,VORNAMEN)) #textacy.fileio.write_file_lines(LEMMAS,"lemmas.txt") #textacy.fileio.write_file_lines(VORNAMEN,"firstnames.txt") textacy.fileio.write_file_lines(de_stop_words,"german_stopwords.txt") """ end = time.time() print("\n\n\nTime Elapsed Topic:{0}\n\n".format(end - start))