607 lines
22 KiB
Python
607 lines
22 KiB
Python
# -*- coding: utf-8 -*-
|
|
|
|
from datetime import datetime
|
|
import csv
|
|
import sys
|
|
from miscellaneous import *
|
|
from datetime import datetime
|
|
import time
|
|
import textacy
|
|
from scipy import *
|
|
|
|
import os
|
|
|
|
csv.field_size_limit(sys.maxsize)
|
|
FILEPATH = os.path.dirname(os.path.realpath(__file__)) + "/"
|
|
|
|
|
|
|
|
# load config
|
|
config_ini = FILEPATH + "config.ini"
|
|
ressources_path = FILEPATH + "ressources/"
|
|
|
|
config = ConfigParser.ConfigParser()
|
|
with open(config_ini) as f:
|
|
config.read_file(f)
|
|
|
|
|
|
|
|
|
|
REGEX_SPECIALCHAR = r'[`\-=~%^&*()_+\[\]{};\'\\:"|</>]' #+r',.'
|
|
REGEX_TOPLVL = r'\.[a-z]{2,3}(\.[a-z]{2,3})?'
|
|
|
|
global THESAURUS
|
|
global WORDS
|
|
global LEMMAS
|
|
global NOUNS
|
|
global VORNAMEN
|
|
global DE_STOP_WORDS
|
|
global EN_STOP_WORDS
|
|
|
|
THESAURUS = {}
|
|
WORDS= {}
|
|
LEMMAS= {}
|
|
NOUNS= {}
|
|
VORNAMEN= {}
|
|
DE_STOP_WORDS= {}
|
|
EN_STOP_WORDS= {}
|
|
|
|
|
|
|
|
############# filter tokens
|
|
|
|
|
|
def filterTokens(tokens, funclist):
|
|
# in:tokenlist, funclist
|
|
# out: tokenlist
|
|
for f in funclist:
|
|
tokens = list(filter(f, tokens))
|
|
|
|
for tok in tokens:
|
|
if tok.pos_ =="NOUN":
|
|
x=0
|
|
|
|
return tokens
|
|
|
|
|
|
def keepPOS(pos_list):
|
|
return lambda tok: tok.pos_ in pos_list
|
|
|
|
|
|
def keepNouns(noun_list=NOUNS):
|
|
#return lambda tok: tok.lower_ in noun_list
|
|
return lambda tok: tok.lower_ in noun_list or tok.pos_ == "NOUN"
|
|
|
|
|
|
|
|
def removePOS(pos_list):
|
|
return lambda tok: tok.pos_ not in pos_list
|
|
|
|
|
|
def removeWords(words, keep=None):
|
|
if hasattr(keep, '__iter__'):
|
|
for k in keep:
|
|
try:
|
|
words.remove(k)
|
|
except ValueError:
|
|
pass
|
|
|
|
return lambda tok: tok.lower_ not in words
|
|
|
|
|
|
def keepENT(ent_list):
|
|
return lambda tok: tok.ent_type_ in ent_list
|
|
|
|
|
|
def removeENT(ent_list):
|
|
return lambda tok: tok.ent_type_ not in ent_list
|
|
|
|
|
|
def remove_words_containing_Numbers():
|
|
return lambda tok: not bool(re.search('\d', tok.lower_))
|
|
|
|
|
|
def remove_words_containing_topLVL():
|
|
return lambda tok: not bool(re.search(REGEX_TOPLVL, tok.lower_))
|
|
|
|
|
|
def remove_words_containing_specialCharacters():
|
|
return lambda tok: not bool(re.search(REGEX_SPECIALCHAR, tok.lower_))
|
|
|
|
|
|
def remove_long_words():
|
|
return lambda tok: not len(tok.lower_) < 2
|
|
|
|
|
|
def remove_short_words():
|
|
return lambda tok: not len(tok.lower_) > 35
|
|
|
|
|
|
def remove_first_names():
|
|
return lambda tok: tok.lower_ not in [name.lower() for name in VORNAMEN]
|
|
|
|
|
|
############# strings
|
|
|
|
def remove_addresses(string):
|
|
pass # todo remove_addresses idee postal.parser und zu metadaten hinzufügen
|
|
|
|
def lemmatizeWord(word,lemma_dict=LEMMAS,n=5):
|
|
for i in range(n):
|
|
try:
|
|
word = lemma_dict[word.lower()] if word.lower() in lemma_dict.keys() else word.lower()
|
|
except:
|
|
print(word)
|
|
return word
|
|
|
|
|
|
def getFirstSynonym(word, thesaurus=THESAURUS, n=3):
|
|
|
|
for i in range(n):
|
|
|
|
try:
|
|
if word in thesaurus.keys():
|
|
return thesaurus[word]
|
|
|
|
elif word.title() in thesaurus.keys():
|
|
return thesaurus[word.title()]
|
|
|
|
elif word.lower() in thesaurus.keys():
|
|
return thesaurus[word.lower()]
|
|
|
|
else:
|
|
return word
|
|
|
|
|
|
|
|
except:
|
|
print("THESAURUSFEHLER BEI: {}".format(word))
|
|
return word
|
|
|
|
|
|
########################## Spellchecking ##########################################
|
|
# http://norvig.com/spell-correct.html
|
|
# http://wortschatz.uni-leipzig.de/en/download
|
|
|
|
import re
|
|
|
|
|
|
def words(text): return re.findall(r'\w+', text.lower())
|
|
|
|
def P(word, N=sum(WORDS.values())):
|
|
"Probability of `word`."
|
|
return WORDS[word] / N
|
|
|
|
|
|
def correction(word):
|
|
"Most probable spelling correction for word."
|
|
return max(candidates(word), key=P)
|
|
|
|
|
|
def candidates(word):
|
|
"Generate possible spelling corrections for word."
|
|
return (known([word]) or known(edits1(word)) or known(edits2(word)) or [word])
|
|
|
|
|
|
def known(words):
|
|
"The subset of `words` that appear in the dictionary of WORDS."
|
|
return set(w for w in words if w in WORDS)
|
|
|
|
|
|
def edits1(word):
|
|
"All edits that are one edit away from `word`."
|
|
letters = 'abcdefghijklmnopqrstuvwxyz'
|
|
splits = [(word[:i], word[i:]) for i in range(len(word) + 1)]
|
|
deletes = [L + R[1:] for L, R in splits if R]
|
|
transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R) > 1]
|
|
replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
|
|
inserts = [L + c + R for L, R in splits for c in letters]
|
|
return set(deletes + transposes + replaces + inserts)
|
|
|
|
|
|
def edits2(word):
|
|
"All edits that are two edits away from `word`."
|
|
return (e2 for e1 in edits1(word) for e2 in edits1(e1))
|
|
|
|
|
|
def autocorrectWord(word):
|
|
try:
|
|
return correction(word)
|
|
except:
|
|
return word
|
|
|
|
|
|
############# stringcleaning
|
|
|
|
|
|
def processContentstream(textstream, parser, token_filterlist=None):
|
|
|
|
#pre parse
|
|
textstream = preparse(textstream)
|
|
|
|
pipe = parser.pipe(textstream)
|
|
|
|
for doc in pipe:
|
|
|
|
tokens = [tok for tok in doc]
|
|
|
|
# in parse
|
|
if token_filterlist is not None:
|
|
tokens = filterTokens(tokens, token_filterlist)
|
|
|
|
# post parse
|
|
#todo STELLSCHRAUBE tokens = [postparse(tok) for tok in tokens] #todo: informationsverlust von pos,tag etc.!
|
|
tokens = [tok.lower_ for tok in tokens]
|
|
yield " ".join(tokens)
|
|
|
|
def preparse(stringstream):
|
|
|
|
for string in stringstream:
|
|
|
|
# cut_after
|
|
words = ["gruss", "grusse","gruesse","gruessen","grusses"]
|
|
|
|
for gr in words:
|
|
if gr in string:
|
|
string = string.rpartition(gr)[0]
|
|
break
|
|
|
|
yield string
|
|
|
|
def postparse(toktext):
|
|
"""
|
|
:param toktext: spacy.token
|
|
:return: string
|
|
"""
|
|
toktext = toktext.lower_
|
|
|
|
# remove_words_containing_topLVL
|
|
toktext = toktext if not re.search(REGEX_TOPLVL, toktext) else ""
|
|
|
|
# lemmatize
|
|
toktext = lemmatizeWord(toktext)
|
|
|
|
# synonyme normalisieren
|
|
toktext = getFirstSynonym(toktext)
|
|
|
|
# autocorrect
|
|
toktext = autocorrectWord(toktext)
|
|
|
|
return toktext
|
|
|
|
|
|
def processDictstream(dictstream, funcdict, parser):
|
|
"""
|
|
|
|
:param dictstream: dict-gen
|
|
:param funcdict:
|
|
clean_in_meta = {
|
|
"Solution":funclist,
|
|
...
|
|
}
|
|
|
|
:param parser: spacy-parser
|
|
:return: dict-gen
|
|
"""
|
|
for dic in dictstream:
|
|
result = {}
|
|
for key, value in dic.items():
|
|
|
|
if key in funcdict:
|
|
|
|
doc = parser(value)
|
|
tokens = [tok for tok in doc]
|
|
funclist = funcdict[key]
|
|
|
|
tokens = filterTokens(tokens, funclist)
|
|
|
|
result[key] = " ".join([tok.lower_ for tok in tokens])
|
|
|
|
|
|
else:
|
|
result[key] = value
|
|
yield result
|
|
|
|
|
|
##################################################################################################
|
|
|
|
|
|
|
|
path2thesaurus_dict = ressources_path + config.get("thesaurus","pickle_file")
|
|
|
|
path2wordsdict = ressources_path + config.get("spellchecking", "pickle_file")
|
|
|
|
path2lemmadict = ressources_path + config.get("lemmatization","pickle_file")
|
|
|
|
path2firstnameslist = ressources_path + config.get("firstnames","pickle_file")
|
|
|
|
|
|
path2DEstopwordlist = ressources_path + config.get("de_stopwords", "pickle_file")
|
|
|
|
path2ENstopwordlist = ressources_path + config.get("en_stopwords", "pickle_file")
|
|
|
|
custom_words = get_list_from_config("preprocessing", "custom_words")
|
|
|
|
|
|
|
|
corpus_de_path = FILEPATH + config.get("de_corpus", "path")
|
|
de_plainpath = FILEPATH + config.get("de_corpus", "path") + "pre_labled_lines.txt"
|
|
|
|
corpus_en_path = FILEPATH + config.get("en_corpus", "path")
|
|
|
|
|
|
def extract_from_corpus(corpus):
|
|
"""
|
|
Extract from each doc from a corpus a string containing disired token_texts
|
|
|
|
|
|
:param corpus: textacy.Corpus
|
|
:return: string-gen
|
|
"""
|
|
|
|
# WHITELIST erstellen. Enthält zumindest die evtuellen Topics
|
|
|
|
WHITELIST = ["boss", "sap", "firefox"] #todo autogenerierung relv. techn. begriffe
|
|
|
|
kb_cats = ['eldorado', 'cws_confluence', 'wsus', 'mail groupware', 'd.3 dms', 'serviceportal', 'softwarelizenzen', 'sophos', 'webserver', 'sap', 'ftp server', 'dhcp', 'tonerboerse', 'mailalias', 'arbeitsplatzsupport', 'mediendienste', 'mailverteiler', 'uni mail', 'basis app', 'videoschnitt', 'DEFAULT', 'verwaltung', 'matrix42_hilfe', 'hoersaaluebertragung', 'redmine', 'uniflow', 'keine rueckantwort', 'pools', 'leitung', 'netze', 'konteneinsicht', 'kennwort aenderung', 'datanet', 'neuanschluss', 'semesterticket', 'asknet', 'veranstaltungen', 'housing', 'fk 16', 'fiona', 'betrieb', 'vorlagenerstellung', 'studierendensekretariat', 'pvp', 'mobilfunkvertraege', 'ausleihe', 'web', 'spam phishing', 'sap urlaub', 'evaexam', 'vorlesungsaufzeichnung', 'firewall betreuung', 'ub basis it', 'virtuelle desktops citrix', 'fk15', 'virtuelle server', 'lizenzserver', 'elektronisches telefonbuch', 'joomla itmc website', 'weiterentwicklung', 'serversupport', 'wlan', 'kurse', 'technik', 'raumkalender', 'backup tsm', 'haustechnik', 'voicemail box', 'facility', 'unicard ausgabe', 'mdm mobile device management', 'entwicklung', 'webgestaltung', 'unicard sperrung', 'forensic', 'basis applikationen', 'overhead projektor', 'plagiatserkennung', 'uniaccount zugangsdaten', 'zentrale webserver', 'webmailer', 'fk12 webauftritt', 'plotter', 'campus management', 'ub_stoerungen', 'rundmail', 'telefon', 'raumbuchung', 'fk12 migration', 'dienstreise', 'hardware', 'it sicherheit sic', 'hochleistungsrechnen', 'unicard', 'sos', 'benutzerverwaltung_probleme', 'confluence', 'vpn', 'zhb', 'campus app', 'itmc_aufgaben', 'sicherheit', 'schulungsraum verwaltung', 'unicard produktion', 'schulung', 'video', 'dokoll support', 'sd', 'servicedesk', 'v2 campus app feedback', 'lido', 'app feedback', 'ibz raumbuchung', 'hcm stammdaten', 'itmc_stoerungen', 'boss service desk', 'exchange nutzung', 'office', 'rektorat -buero', 'bestellung', 'moodle', 'fk raumplanung 09', 'aenderung', 'neuausstattung', 'benutzerverwaltung', 'rechnerraeume', 'designentwicklung', 'fk 12', 'werkstoffe lehrstuhl bauwesen', 'server storage', 'beantragung', 'visitenkartenproduktion', 'gastaufenthalt', 'telefonkonferenzen', 'raumbuchungssysteme', 'fk14_test', 'e mail dienste', 'grafik', 'ews', 'itmc schulungsraeume', 'tsm', 'softwareverteilung', 'beamer', 'lizenzmanagement', 'fileserver einrichtung', 'redmine projektverwaltung', 'service desk itmc', 'pruefungsmanagement', 'prozess- und projektmanagement', 'formulare antraege', 'namensaenderung', 'verkauf', 'software', 'itmc medienraeume ef50', 'zugangsdaten', 'medientechnik', 'lan', 'veeam', 'unicard redaktionsteam', 'changes', 'service portal', 'limesurvey', 'dns', 'dokoll pvp', 'uhren', 'nrw ticket', 'itmc_als', 'linux bs', 'werkvertraege', 'blogs wikis foren', 'test', 'abmeldung', 'desktop & basisdienste', 'telefonzentrale', 'siport zugangskontrolle', 'antrag auf rechnungserstellung', 'verschiedene aufgaben', 'kundenserver', 'medienraeume ef50', 'videokonferenzen', 'benutzungsverwaltung', 'mailverteiler exchange', 'lsf', 'telefonabrechnung', 'werkstaette', 'uniaccount', 'outlook_einrichtung', 'itmc webauftritt', 'zertifikate server dfn', 'allgemein', 'umzug', 'service portal redaktion', 'pos', 'beschaffung', 'boss', 'hacker angriff', 'software entwicklung', 'cd dvd produktion', 'sam spider', 'viren', 'kursplanung', 'itmc pools', 'kms', 'e learning']
|
|
kb_keys = ['zugriff_onlinedienste_rueckmeldung', 'uniaccount', 'freischaltung', 'asknet', 'eduroam', 'donnerstagsmail namensaenderung', 'asiexception', 'lsf', 'kundenantwort', 'chip', 'unitymedia', 'citavi', 'fehler', 'windows beziehen', 'wlan', 'ipv6', 'freischaltung verzoegert', 'betrag', '"defekte karte"', 'risse', 'laden', 'sap portal anderer modus', 'goeke', 'informationen des itmc zum einsatz', 'transport wurde durchgefuehrt.', 'wi-fi', 'unicard_auszahlung', 'ausleihe', 'unimail', 'uni-account', 'unicard','beantragung', 'nrw-ticket', 'printservice', 'dms', 'ip6', 'transport und beschreibung zum transportauftrag !', 'wlan passwort', 'dokumentenmanagementsystem', 'webmailer', 'vpn', 'repository', 'unicard', 'projekte', 'eingeschrieben', 'unicard abholung oeffnungszeiten', 'd3', 'beantragung', 'app tu-dortmund feedback', 'semester ticket', 'redmine', 'git', 'geldkarte', 'outlook_exchange', 'spam standardmeldung phishing', 'automatische aktualisierung der selbst angelegten kontakte in outlook', '"beschaedigte unicard"', 'elektronische telefonbuch', 'boss', 'wwrite', 'DEFAULT', 'anyconnect', 'wifi']
|
|
kb_subjs =['sd_office 365 plus support', 'citavi_lizenzschluessel_nicht bekommen', 'uni card', 'sd_office 356 plus bestellung', 'sd_gastaufenthalter', 'sd_outlook kontakte automatische aktualisierung', 'benutzer zum redmine hinzufuegen', 'sd_matlab lizenzdatei pc-pools', 'sd_tu-app feedback standard', 'vpn_ipsec_stoerung', 'vpn verbindung fuer unitymedia kunden', 'ub_prod_abholung_ abholfristen_benachrichtigungen', 'einrichtung des eduroam netzwerks', 'sd_webmailer_threadanzeige und weiterleitung', 'sd_wlan passwort setzen', 'ub_prod_namenskorrektur_student', 'sd_unimail imap_pop3', 'sd_outlook_in_exchange_einbinden', 'sd_keine rueckantwort kunde', 'sd_asknet_und_dreamspark', 'sd_heirat_namensaenderung_student', 'bd_unicard_nicht_eingeschrieben', 'wlan', 'sd_telefonbuch_prof_eintragung', 'change produktiv nehmen chn00146 - transport e01k909284', 'ungueltiges ticket siehe journal', 'apps_dms_d.3 client installation/login d.3 funktioniert nicht', 'd.3 client installation', 'unicard_restbetrag_auszahlung', 'cm_asiexception', 'sd_origin_workaround', 'sd_vpn_aktualisierung', 'problem mit der beantragung von der unicard', 'sd_unicard fehlerhafte geldbuchung', 'sd_login tu portals english', 'sd_gmx_web.de', 'studierendenausweis', 'sd_citavi', 'sd_fk9 test', 'sd_webmailer_thread-anzeige', 'bd_unicard_geldkarte_laden', 'ub_unicard_unicard mit vollmacht abholen', 'sd_stellenausschreibung schwarzes brett', 'freischaltung uniaccount', 'sd_asknet_mitarbeiter_softwarebestellung', 'how to setup eduroam', 'sd_citavi bestellung', 'unicard vergessen abzuholen und nicht mehr da', 'sd_unimail zu exchange', 'sd_diensthandy beschaffung', 'sd_sap konteneinsicht antrag', 'sd_unicard_defekt', 'sd_webmailer einrichtung weiterleitung', 'sd_kurs-angebote anmeldung', 'm42_dokumentationen_zu_neuen_ous', 'sd_sap_initialkennwort', 'sd_sap_freischaltung ohne passwortaenderung', 'sd_telefonbuch-eintrag_aenderung', 'sd_pruefungsamt', 'sd_phishing', 'apps_dms-passwort d.3', 'sd_goeke drucker', 'sd_sap_dienstreise', 'unicard nochmal beantragen', 'sd_outlook anmeldung gestoert', 'sd_citavi_support', 'DEFAULT', 'sd_geraeteausleihe', 'sd_account_abmelden', 'sd_uniaccount freischaltung verzoegert englisch', 'ub_beschaedigte unicard', 'sd_gleitzeitanlage_dez3_stoerung', 'transportdurchfuehung', 'sd_sap_initialkennwort_englisch', 'sd_antwort_phishingmail', 'sd_namensaenderung mitarbeiter', 're: elektroarbeiten fuer leitsystem 2. und 3. obergeschoss', 'lsf freischaltung als mitarbeiter/in', 'ub_unicard_spaetere abholung moeglich?', 'sd_antrag funktionale mailadresse', 'sd_apple-on-campus', 'sd_office365_asknet', 'sd_sophos download', 'sd_freischaltung uniaccount verzoegert', 'ub_unicard_zusendung der karte moeglich?', 'ohne betreff', 'sd_immatrikulationsbescheinigung_portal', 'sd_studisek_buchung_semesterbeitrag', 'sd_studisek_englisch', 'probleme mit der namensaenderung/ neue unicard', 'sd_telefonbuch, neues system', 'fehlender eintrag im elektronischen telefonbuch', 'sd_boss_notenverbuchung', 'sd_laufzeit unimail account', 'sd_semesterticket', 'sd_kontakt_asknet', 'windows 10', 'sd_login_tu_portale', 'ub_geldchip-problem bei uc', 'sd_zugriff_onlinedienste_rueckmeldung', 'sd_wlan-gastkonto', 'sd_tu_app_keine internetverbindung', 'sd_uniaccount_ehemalige_passwortaenderung', 'sd_verlust/antrag unicard', 'sd_sap_konteneinsicht_ workaround', 'apps_redmine_repository', 'sd_itmc kurse anmeldebestaetigung', 'sd_mail_als_anhang', 'bd_unicard_chip_defekt', 'probleme mit unicard', 'ub_unicard_abholungszeiten', 'sd_falsche_personendaten', 'sd_uniaccount_ehemalige_studierende', 'sd_vpn anleitungen', 'sd_kurs-angebote itmc', 'sd_studisek', 'sd_login tu portale', 'sd_immatrikulationsbescheigung_druckfehler', 'ub_drucker kopierer', 'sd_vpn_temporaerer fehler ub', 'sd_spss_online_bestellung', 'sd_dreamspark', 'sd_unicard_gesperrte unicard entsperre', 'sd_boss-bescheinigung', 'bd_goeke_allgemein', 'sd_uniaccount_passwortaenderung', 'sd_namensaenderung_englisch', 'sd_email_namensaenderung', 'bd_unicard_freigabe_beantragung', 'spam ohne tu bezug', 'sd_internationaloffice', 'sd_tu-app feedback_englisch', 'cm_lsf-boss_freischaltung', 'sd-e-mail_adresse_funktional_beantragen', 'sd_vpn_webvpn', 'sd_vpn_probleme_mit_unitymedia', 'sd_plotauftrag_zv', 'sd_beantragung_unicard', 'sd_antworten_korrekt', 'ub_prod_neue unicard bei beschaedigung', 'sd_telefonantrag_aenderung_neuantrag', 'sd_wlan passwort englisch', 'sd_aktivierung uniaccount', 'sd_spam e-mail bekannt meldung', 'sd_wlan_beratung', 'ub_namensaenderung', 'sd_telefon (antrag: neuanschluss, umzug, aenderung erledigt)', 'sd_unicard_abholung', 'sd_uniaccount_dauer freischaltung', 'sd_uniaccount activation englisch', 'sd_unicard_max_laufzeit', 'sd_unicard_workaround_bestellung', 'sd_sap_firefox_esr', 'sap portal "im anderen modus geoeffnet"', 'sd_origin nur noch eine seriennummer', 'sd_login_unibib ub-it']
|
|
|
|
WHITELIST = WHITELIST + kb_cats + kb_keys + kb_subjs
|
|
|
|
|
|
THESAURUS = load_obj(path2thesaurus_dict)
|
|
#WORDS = load_obj(path2wordsdict)
|
|
LEMMAS = load_obj(path2lemmadict)
|
|
DE_STOP_WORDS = load_obj(path2DEstopwordlist)
|
|
#EN_STOP_WORDS = load_obj(path2ENstopwordlist)
|
|
VORNAMEN = load_obj(path2firstnameslist)
|
|
|
|
ents_boss = []
|
|
ents_sap = []
|
|
|
|
for doc in corpus:
|
|
result = []
|
|
|
|
#if doc.metadata["TicketNumber"] == "INC40506":
|
|
# breakpoint()
|
|
|
|
|
|
for tok in doc:
|
|
|
|
|
|
"""
|
|
if tok.lower_ =="boss":
|
|
ents_boss.append(tok.ent_type_)
|
|
|
|
if tok.lower_ =="sap":
|
|
ents_sap.append(tok.ent_type_)
|
|
"""
|
|
|
|
# wenn in whitelist, direkt übernehmen
|
|
if tok.lower_ in WHITELIST:
|
|
result.append(tok.lower_)
|
|
|
|
|
|
# ignore header, urls , emails, stop, vornamen
|
|
lemmatized_word = lemmatizeWord(tok.text,lemma_dict=LEMMAS)
|
|
if lemmatized_word.lower() in ["sehr", "geehrt", "herr" ,"herrn", "herren", "dame" , "damen", "liebe","lieben", "hallo", "guten", "tag","ehre","hi"] \
|
|
or tok.like_url \
|
|
or tok.like_email \
|
|
or tok.is_stop \
|
|
or tok.is_punct \
|
|
or tok.lower_ in DE_STOP_WORDS \
|
|
or tok.lower_ in VORNAMEN:
|
|
continue
|
|
|
|
|
|
|
|
# cut after footer
|
|
if replaceRockDots(tok.lower_) in ["gruss", "grusse", "gruesse", "gruessen", "grusses"]: # fehler schneidet bei zB INC40506 das meiste weg
|
|
break
|
|
|
|
|
|
|
|
if tok.pos_ in ["NOUN"] \
|
|
or tok.ent_type_ in ["NORP","FACILITY","ORG","PRODUCT","WORK_OF_ART","LOC"]:
|
|
#or tok.dep_ == "ROOT":
|
|
# or tok.lower_ in NOUNS \ #,"PERSON"] \
|
|
toktext = tok.lower_
|
|
|
|
|
|
toktext = lemmatized_word
|
|
|
|
# hauptsynonym bilden idee zwar das Huaptsyn bilden und zählen aber die originalen wörter in den llda algo geben
|
|
"""
|
|
first_synonym = getFirstSynonym(toktext, thesaurus=THESAURUS)
|
|
if first_synonym is not None or first_synonym != '':
|
|
toktext = first_synonym if len(first_synonym.split()) == 1 else toktext
|
|
"""
|
|
|
|
|
|
result.append(toktext)
|
|
|
|
|
|
|
|
yield " ".join(result)
|
|
|
|
|
|
"""
|
|
print(list(set(ents_sap)))
|
|
['', 'ORG', 'PERSON', 'LOC']
|
|
|
|
print(list(set(ents_boss)))
|
|
['', 'ORG', 'PERSON', 'LOC']
|
|
"""
|
|
|
|
|
|
|
|
|
|
def preprocessCorpus(corpus, clean_in_meta):
|
|
|
|
logprint("Preprocess {0}_corpus at {1}".format(corpus.lang, datetime.now()))
|
|
|
|
|
|
preCorpus_name = corpus.lang + "_pre"
|
|
|
|
|
|
clean_corpus = corpus
|
|
|
|
parser = corpus.spacy_lang
|
|
|
|
pre_corpus = textacy.Corpus(parser)
|
|
|
|
|
|
|
|
|
|
## process and add files to textacy-corpi,
|
|
pre_corpus.add_texts(
|
|
|
|
|
|
#processContentstream(corpus2Text(clean_corpus), token_filterlist=filter_tokens, parser=parser),
|
|
extract_from_corpus(clean_corpus),
|
|
processDictstream(corpus2Meta(clean_corpus), clean_in_meta,parser=parser)
|
|
|
|
|
|
)
|
|
|
|
# idee labeled_lines.txt enthählt bigramme mit unterstrich
|
|
# todo preCorpus weg. llda bekommt labaled_lines.txt und lda doctermamtrix
|
|
|
|
|
|
# leere docs aus corpi kicken
|
|
pre_corpus.remove(lambda doc: len(doc) == 0)
|
|
|
|
|
|
#save corpus
|
|
save_corpus(corpus=pre_corpus, corpus_path=corpus_de_path, corpus_name=preCorpus_name)
|
|
|
|
|
|
#save corpus as labled, plain text
|
|
savelabledCorpiLines(pre_corpus, de_plainpath)
|
|
|
|
labled_lines =""
|
|
return pre_corpus
|
|
|
|
|
|
def main(corpus):
|
|
start = time.time()
|
|
|
|
|
|
|
|
|
|
|
|
"""
|
|
filter_tokens = [
|
|
|
|
keepNouns(NOUNS),
|
|
|
|
removeWords(DE_STOP_WORDS + custom_words + VORNAMEN),
|
|
|
|
removePOS(["PUNCT", "SPACE", "NUM"]),
|
|
|
|
#todo STELLSCHRAUBE remove_words_containing_Numbers(),
|
|
#todo STELLSCHRAUBE remove_long_words(),
|
|
#todo STELLSCHRAUBE remove_short_words()
|
|
|
|
]
|
|
"""
|
|
clean_in_meta = {
|
|
"Solution": [removePOS(["SPACE"])],
|
|
"Subject": [removePOS(["SPACE", "PUNCT"])],
|
|
"categoryName": [removePOS(["SPACE", "PUNCT"])]
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
pre_corpus = preprocessCorpus(corpus, clean_in_meta)
|
|
|
|
|
|
|
|
|
|
#for i in range(5):
|
|
# printRandomDoc(pre_corpus)
|
|
|
|
|
|
|
|
|
|
end = time.time()
|
|
logprint("Time Elapsed Preprocessing:{0} min".format((end - start) / 60))
|
|
return pre_corpus
|
|
|
|
if __name__ == "__main__":
|
|
corpus, parser = load_corpus(corpus_path="/home/jannis.grundmann/PycharmProjects/topicModelingTickets/corpi/",corpus_name="de_clean")
|
|
|
|
|
|
main(corpus)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
"""
|
|
pipe=[
|
|
|
|
##String
|
|
|
|
fixUnicode(),
|
|
replaceHardS(),
|
|
resolveAbbrivations(),
|
|
|
|
remove_words_containing_topLVL(),
|
|
|
|
replaceSpecialChars(" "), (mit Leerzeichen erstzen, dadruch werden Terme wie 8203;verfügung getrennt
|
|
|
|
remove_words_containing_Numbers(),
|
|
|
|
|
|
|
|
##spacyParse
|
|
|
|
removeENT("PERSON"),
|
|
keepPOS(["NOUN"]),
|
|
|
|
#ODER
|
|
|
|
lemmatize(),
|
|
removeWords(de_stop_words + config.get("preprocessing","custom_words").split(",")),
|
|
|
|
|
|
# evtl.
|
|
spellCorrection(),
|
|
keepUniqeTokens(),
|
|
|
|
]
|
|
|
|
|
|
|
|
|
|
"""
|
|
|
|
"""
|
|
filter_tokens=[
|
|
#removeENT(["PERSON"]),
|
|
#idee addressen enfernen #bisher mit cut_after("gruss") --> postal.parser
|
|
#idee rechtschreibkorrektur --> PyEnchant
|
|
#idee thesaurus --> WordNet, eigener
|
|
|
|
remove_words_containing_Numbers(),
|
|
|
|
removePOS(["PUNCT","SPACE","NUM"]),
|
|
|
|
removeWords(de_stop_words+custom_words),
|
|
|
|
remove_long_words(),
|
|
remove_short_words(),
|
|
remove_first_names(),
|
|
|
|
keepPOS(["NOUN"]),
|
|
|
|
]
|
|
"""
|