textcleaning macht keinen spass

backup.py

@@ -0,0 +1,383 @@
+# -*- coding: utf-8 -*-
+
+
+############# misc
+
+def printlog(string, level="INFO"):
+    """log and prints"""
+    print(string)
+    if level=="INFO":
+        logging.info(string)
+    elif level=="DEBUG":
+        logging.debug(string)
+    elif level == "WARNING":
+        logging.warning(string)
+printlog("Load functions")
+
+def compose(*functions):
+    def compose2(f, g):
+        return lambda x: f(g(x))
+    return functools.reduce(compose2, functions, lambda x: x)
+
+def get_calling_function():
+    """finds the calling function in many decent cases.
+    https://stackoverflow.com/questions/39078467/python-how-to-get-the-calling-function-not-just-its-name
+    """
+    fr = sys._getframe(1)   # inspect.stack()[1][0]
+    co = fr.f_code
+    for get in (
+        lambda:fr.f_globals[co.co_name],
+        lambda:getattr(fr.f_locals['self'], co.co_name),
+        lambda:getattr(fr.f_locals['cls'], co.co_name),
+        lambda:fr.f_back.f_locals[co.co_name], # nested
+        lambda:fr.f_back.f_locals['func'],  # decorators
+        lambda:fr.f_back.f_locals['meth'],
+        lambda:fr.f_back.f_locals['f'],
+        ):
+        try:
+            func = get()
+        except (KeyError, AttributeError):
+            pass
+        else:
+            if func.__code__ == co:
+                return func
+    raise AttributeError("func not found")
+
+
+def printRandomDoc(textacyCorpus):
+    import random
+    print()
+
+    printlog("len(textacyCorpus) = %i" % len(textacyCorpus))
+    randIndex = int((len(textacyCorpus) - 1) * random.random())
+    printlog("Index: {0} ; Text: {1} ; Metadata: {2}".format(randIndex, textacyCorpus[randIndex].text, textacyCorpus[randIndex].metadata))
+
+    print()
+
+#############  load xml
+def generateMainTextfromTicketXML(path2xml, main_textfield='Description'):
+    """
+    generates strings from XML
+    :param path2xml:
+    :param main_textfield:
+    :param cleaning_function:
+    :yields strings
+    """
+    tree = ET.parse(path2xml, ET.XMLParser(encoding="utf-8"))
+    root = tree.getroot()
+
+    for ticket in root:
+        for field in ticket:
+            if field.tag == main_textfield:
+                    yield field.text
+
+def generateMetadatafromTicketXML(path2xml, leave_out=['Description']):
+    tree = ET.parse(path2xml, ET.XMLParser(encoding="utf-8"))
+    root = tree.getroot()
+
+    for ticket in root:
+        metadata = {}
+        for field in ticket:
+            if field.tag not in leave_out:
+
+                    metadata[field.tag] = field.text
+
+        yield metadata
+
+
+#############  load csv
+
+def csv_to_contentStream(path2csv: str, content_collumn_name: str):
+    """
+    :param path2csv: string
+    :param content_collumn_name: string
+    :return: string-generator
+    """
+    stream = textacy.fileio.read_csv(path2csv, delimiter=";")  # ,encoding='utf8')
+    content_collumn = 0  # standardvalue
+
+    for i,lst in enumerate(stream):
+        if i == 0:
+            # look for desired column
+            for j,col in enumerate(lst):
+                if col == content_collumn_name:
+                    content_collumn = j
+        else:
+            yield lst[content_collumn]
+
+def csv_to_metaStream(path2csv: str, metalist: [str]):
+    """
+    :param path2csv: string
+    :param metalist: list of strings
+    :return: dict-generator
+    """
+    stream = textacy.fileio.read_csv(path2csv, delimiter=";")  # ,encoding='utf8')
+
+    content_collumn = 0  # standardvalue
+    metaindices = []
+    metadata_temp = {}
+    for i,lst in enumerate(stream):
+        if i == 0:
+            for j,col in enumerate(lst):        # geht bestimmt effizienter... egal, weil passiert nur einmal
+                for key in metalist:
+                    if key == col:
+                        metaindices.append(j)
+            metadata_temp = dict(zip(metalist,metaindices)) # zB {'Subject' : 1, 'categoryName' : 3, 'Solution' : 10}
+
+        else:
+            metadata = metadata_temp.copy()
+            for key,value in metadata.items():
+                metadata[key] = lst[value]
+            yield metadata
+
+
+
+############################################    Preprocessing   ##############################################
+
+
+#############  on str-gen
+
+def processTokens(tokens, funclist, parser):
+    # in:tokenlist, funclist
+    # out: tokenlist
+    for f in funclist:
+        # idee: funclist sortieren,s.d. erst alle string-methoden ausgeführt werden, dann wird geparesed, dann wird auf tokens gearbeitet, dann evtl. auf dem ganzen Doc
+
+        if 'bool' in str(f.__annotations__):
+            tokens = list(filter(f, tokens))
+
+        elif 'str' in str(f.__annotations__):
+            tokens = list(map(f, tokens))  # purer text
+            doc = parser(" ".join(tokens))  # neu parsen
+            tokens = [tok for tok in doc]  # nur tokens
+
+        elif 'spacy.tokens.doc.Doc' in str(f.__annotations__):
+            #todo wirkt gefrickelt
+            doc = parser(" ".join(tok.lower_ for tok in tokens))  # geparsed
+            tokens = f(doc)
+            doc = parser(" ".join(tokens))  # geparsed
+            tokens = [tok for tok in doc]  # nur tokens
+        else:
+            warnings.warn("Unknown Annotation while preprocessing. Function: {0}".format(str(f)))
+
+    return tokens
+
+def processTextstream(textstream, funclist, parser=DE_PARSER):
+    """
+    :param textstream: string-gen
+    :param funclist: [func]
+    :param parser: spacy-parser
+    :return: string-gen
+    """
+    # input:str-stream output:str-stream
+    pipe = parser.pipe(textstream)
+
+    for doc in pipe:
+
+        tokens = []
+        for tok in doc:
+            tokens.append(tok)
+
+        tokens = processTokens(tokens,funclist,parser)
+        yield " ".join([tok.lower_ for tok in tokens])
+
+def processDictstream(dictstream, funcdict, parser=DE_PARSER):
+    """
+
+    :param dictstream: dict-gen
+    :param funcdict:
+                    clean_in_meta = {
+                        "Solution":funclist,
+                        ...
+                    }
+
+    :param parser: spacy-parser
+    :return: dict-gen
+    """
+    for dic in dictstream:
+        result = {}
+        for key, value in dic.items():
+
+            if key in funcdict:
+
+                doc = parser(value)
+                tokens = [tok for tok in doc]
+                funclist = funcdict[key]
+
+                tokens = processTokens(tokens,funclist,parser)
+
+
+                result[key] = " ".join([tok.lower_ for tok in tokens])
+
+
+            else:
+                result[key] = value
+        yield result
+
+
+#############  return bool
+
+def keepPOS(pos_list) -> bool:
+    ret = lambda tok : tok.pos_ in pos_list
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+def removePOS(pos_list)-> bool:
+    ret = lambda tok : tok.pos_ not in pos_list
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+def removeWords(words, keep=None)-> bool:
+    if hasattr(keep, '__iter__'):
+        for k in keep:
+            try:
+                words.remove(k)
+            except ValueError:
+                pass
+
+    ret = lambda tok :  tok.lower_ not in words
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+def keepENT(ent_list) -> bool:
+    ret = lambda tok : tok.ent_type_ in ent_list
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+def removeENT(ent_list) -> bool:
+    ret = lambda tok: tok.ent_type_ not in ent_list
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+def remove_words_containing_Numbers() -> bool:
+    ret = lambda tok: not bool(re.search('\d', tok.lower_))
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+
+def remove_words_containing_specialCharacters() -> bool:
+    ret = lambda tok: not bool(re.search(r'[`\-=~!@#$%^&*()_+\[\]{};\'\\:"|<,./<>?]', tok.lower_))
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+
+def remove_words_containing_topLVL() -> bool:
+    ret = lambda tok: not bool(re.search(r'\.[a-z]{2,3}(\.[a-z]{2,3})?', tok.lower_))
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+
+def lemmatizeWord(word,filepath=LEMMAS):
+    """http://www.lexiconista.com/datasets/lemmatization/"""
+    for line in list(textacy.fileio.read_file_lines(filepath=filepath)):
+        if word.lower() == line.split()[1].strip().lower():
+            return line.split()[0].strip().lower()
+    return word.lower() # falls nix gefunden wurde
+
+def lemmatize() -> str:
+    ret = lambda tok: lemmatizeWord(tok.lower_)
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+
+#############  return strings
+
+mentionFinder = re.compile(r"@[a-z0-9_]{1,15}", re.IGNORECASE)
+emailFinder = re.compile(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.IGNORECASE)
+urlFinder = re.compile(r"^(?:https?:\/\/)?(?:www\.)?[a-zA-Z0-9./]+$", re.IGNORECASE)
+topLVLFinder = re.compile(r'\.[a-z]{2,3}(\.[a-z]{2,3})?', re.IGNORECASE)
+specialFinder = re.compile(r'[`\-=~!@#$%^&*()_+\[\]{};\'\\:"|<,./>?]', re.IGNORECASE)
+hardSFinder = re.compile(r'[ß]', re.IGNORECASE)
+
+
+
+def replaceEmails(replace_with="EMAIL") -> str:
+    ret = lambda tok : emailFinder.sub(replace_with, tok.lower_)
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+def replaceURLs(replace_with="URL") -> str:
+    ret = lambda tok: textacy.preprocess.replace_urls(tok.lower_,replace_with=replace_with)
+    #ret = lambda tok: urlFinder.sub(replace_with,tok.lower_)
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+def replaceSpecialChars(replace_with=" ") -> str:
+    ret = lambda tok: specialFinder.sub(replace_with,tok.lower_)
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+
+def replaceTwitterMentions(replace_with="TWITTER_MENTION") -> str:
+    ret = lambda tok : mentionFinder.sub(replace_with,tok.lower_)
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+def replaceNumbers(replace_with="NUMBER") -> str:
+    ret = lambda tok: textacy.preprocess.replace_numbers(tok.lower_, replace_with=replace_with)
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+def replacePhonenumbers(replace_with="PHONENUMBER") -> str:
+    ret = lambda tok: textacy.preprocess.replace_phone_numbers(tok.lower_, replace_with=replace_with)
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+def replaceHardS(replace_with="ss") -> str:
+    ret = lambda tok: hardSFinder.sub(replace_with,tok.lower_)
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+
+def fixUnicode() -> str:
+    ret = lambda tok: textacy.preprocess.fix_bad_unicode(tok.lower_, normalization=u'NFC')
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+
+
+
+
+
+
+
+
+
+def resolveAbbreviations():
+    pass #todo
+
+#todo wörter mit len < 2 entfernen( vorher abkürzungen (v.a. tu und fh) auflösen) und > 35 oder 50 ("Reiserücktrittskostenversicherung)
+
+#############  return docs
+
+def keepUniqeTokens() -> spacy.tokens.Doc:
+    ret = lambda doc: (set([tok.lower_ for tok in doc]))
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+def lower() -> spacy.tokens.Doc:
+    ret = lambda doc: ([tok.lower_ for tok in doc])
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+
+################################################################################################################

spell.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+
+# https://github.com/norvig/pytudes/blob/master/spell.py
+
+"""Spelling Corrector in Python 3; see http://norvig.com/spell-correct.html
+
+Copyright (c) 2007-2016 Peter Norvig
+MIT license: www.opensource.org/licenses/mit-license.php
+"""
+
+################ Spelling Corrector
+
+import re
+from collections import Counter
+import spacy
+import textacy
+def words(text): return re.findall(r'\w+', text.lower())
+
+
+
+
+WORDS = Counter(words(open('bigo.txt').read()))
+x=0
+def P(word, N=sum(WORDS.values())):
+    "Probability of `word`."
+    return WORDS[word] / N
+
+
+def correction(word):
+    "Most probable spelling correction for word."
+    return max(candidates(word), key=P)
+
+
+def candidates(word):
+    "Generate possible spelling corrections for word."
+    return (known([word]) or known(edits1(word)) or known(edits2(word)) or [word])
+
+
+def known(words):
+    "The subset of `words` that appear in the dictionary of WORDS."
+    return set(w for w in words if w in WORDS)
+
+
+def edits1(word):
+    "All edits that are one edit away from `word`."
+    letters = 'abcdefghijklmnopqrstuvwxyz'
+    splits = [(word[:i], word[i:]) for i in range(len(word) + 1)]
+    deletes = [L + R[1:] for L, R in splits if R]
+    transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R) > 1]
+    replaces = [L + c + R[1:] for L, R in splits if R for c in letters]
+    inserts = [L + c + R for L, R in splits for c in letters]
+    return set(deletes + transposes + replaces + inserts)
+
+
+def edits2(word):
+    "All edits that are two edits away from `word`."
+    return (e2 for e1 in edits1(word) for e2 in edits1(e1))
+

test.py

@@ -196,12 +196,14 @@ def processTokens(tokens, funclist, parser):
     # in:tokenlist, funclist
     # out: tokenlist
     for f in funclist:
+        # idee: funclist sortieren,s.d. erst alle string-methoden ausgeführt werden, dann wird geparesed, dann wird auf tokens gearbeitet, dann evtl. auf dem ganzen Doc
+
         if 'bool' in str(f.__annotations__):
             tokens = list(filter(f, tokens))
 
         elif 'str' in str(f.__annotations__):
             tokens = list(map(f, tokens))  # purer text
-            doc = parser(" ".join(tokens))  # geparsed
+            doc = parser(" ".join(tokens))  # neu parsen
             tokens = [tok for tok in doc]  # nur tokens
 
         elif 'spacy.tokens.doc.Doc' in str(f.__annotations__):
@@ -320,6 +322,14 @@ def remove_words_containing_specialCharacters() -> bool:
     ret.__annotations__ = get_calling_function().__annotations__
     return ret
 
+
+def remove_words_containing_topLVL() -> bool:
+    ret = lambda tok: not bool(re.search(r'\.[a-z]{2,3}(\.[a-z]{2,3})?', tok.lower_))
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+
 def lemmatizeWord(word,filepath=LEMMAS):
     """http://www.lexiconista.com/datasets/lemmatization/"""
     for line in list(textacy.fileio.read_file_lines(filepath=filepath)):
@@ -334,58 +344,16 @@ def lemmatize() -> str:
     return ret
 
 
-
-
-
-
-def normalizeSynonyms(default_return_first_Syn=False) -> str:
-    ret = lambda tok : getFirstSynonym(tok.lower_, default_return_first_Syn=default_return_first_Syn)
-
-    ret.__annotations__ = get_calling_function().__annotations__
-    return ret
-
-def getFirstSynonym(word, thesaurus=THESAURUS, default_return_first_Syn=False):
-    if not isinstance(word, str):
-        return str(word)
-
-    word = word.lower()
-
-    # durch den thesaurrus iterieren
-    for syn_block in thesaurus:  # syn_block ist eine liste mit Synonymen
-
-        for syn in syn_block:
-            syn = syn.lower()
-            if re.match(r'\A[\w-]+\Z', syn):  # falls syn einzelwort ist
-                if word == syn:
-                    return str(getHauptform(syn_block, word, default_return_first_Syn=default_return_first_Syn))
-            else:  # falls es ein satz ist
-                if word in syn:
-                    return str(getHauptform(syn_block, word, default_return_first_Syn=default_return_first_Syn))
-    return str(word)  # zur Not, das ursrpüngliche Wort zurückgeben
-
-def getHauptform(syn_block, word, default_return_first_Syn=False):
-    for syn in syn_block:
-        syn = syn.lower()
-
-        if "hauptform" in syn and len(syn.split(" ")) <= 2:
-            # nicht ausgeben, falls es in Klammern steht#todo gibts macnmal?? klammern aus
-            for w in syn.split(" "):
-                if not re.match(r'\([^)]+\)', w):
-                    return w
-
-    if default_return_first_Syn:
-        # falls keine hauptform enthalten ist, das erste Synonym zurückgeben, was kein satz ist und nicht in klammern steht
-        for w in syn_block:
-            if not re.match(r'\([^)]+\)', w):
-                return w
-    return word  # zur Not, das ursrpüngliche Wort zurückgeben
-
-
 #############  return strings
 
 mentionFinder = re.compile(r"@[a-z0-9_]{1,15}", re.IGNORECASE)
 emailFinder = re.compile(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.IGNORECASE)
 urlFinder = re.compile(r"^(?:https?:\/\/)?(?:www\.)?[a-zA-Z0-9./]+$", re.IGNORECASE)
+topLVLFinder = re.compile(r'\.[a-z]{2,3}(\.[a-z]{2,3})?', re.IGNORECASE)
+specialFinder = re.compile(r'[`\-=~!@#$%^&*()_+\[\]{};\'\\:"|<,./>?]', re.IGNORECASE)
+hardSFinder = re.compile(r'[ß]', re.IGNORECASE)
+
+
 
 def replaceEmails(replace_with="EMAIL") -> str:
     ret = lambda tok : emailFinder.sub(replace_with, tok.lower_)
@@ -400,6 +368,13 @@ def replaceURLs(replace_with="URL") -> str:
     ret.__annotations__ = get_calling_function().__annotations__
     return ret
 
+def replaceSpecialChars(replace_with=" ") -> str:
+    ret = lambda tok: specialFinder.sub(replace_with,tok.lower_)
+
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
+
+
 def replaceTwitterMentions(replace_with="TWITTER_MENTION") -> str:
     ret = lambda tok : mentionFinder.sub(replace_with,tok.lower_)
 
@@ -418,7 +393,11 @@ def replacePhonenumbers(replace_with="PHONENUMBER") -> str:
     ret.__annotations__ = get_calling_function().__annotations__
     return ret
 
+def replaceHardS(replace_with="ss") -> str:
+    ret = lambda tok: hardSFinder.sub(replace_with,tok.lower_)
 
+    ret.__annotations__ = get_calling_function().__annotations__
+    return ret
 
 
 def fixUnicode() -> str:
@@ -428,11 +407,20 @@ def fixUnicode() -> str:
     return ret
 
 
+
+
+
+
+
+
+
+
 def resolveAbbreviations():
     pass #todo
 
+#todo wörter mit len < 2 entfernen( vorher abkürzungen (v.a. tu und fh) auflösen) und > 35 oder 50 ("Reiserücktrittskostenversicherung)
 
-#############  return docs #todo das stimmt nicht so ganz, da kommt kein doc raus, sondern n tokenset
+#############  return docs
 
 def keepUniqeTokens() -> spacy.tokens.Doc:
     ret = lambda doc: (set([tok.lower_ for tok in doc]))
@@ -462,7 +450,6 @@ printlog("CSV: {0}".format(path2csv))
 ticketcorpus = textacy.Corpus(DE_PARSER)
 
 
-#idee ß zu ss ändern? prinzipiell?
 
 """
 vllt kategorien in unterkategorien aufteilen 
@@ -503,20 +490,26 @@ clean_in_meta = {
 printlog("Start Preprocessing")
 
 clean_in_content=[
+    replaceHardS(),
+    replaceSpecialChars(),
 
-    removePOS(["SPACE","PUNCT","NUM"]),
+    remove_words_containing_topLVL(),
     remove_words_containing_Numbers(),
     remove_words_containing_specialCharacters(),
 
+    #removePOS(["SPACE","PUNCT","NUM"]),
+    #removeENT("PERSON"),
+
+    #keepPOS(["NOUN"]),
+
+
     #replaceURLs(),
     #replaceEmails(),
     #fixUnicode(),
 
-    #lemmatize(),
+    lemmatize(),
-    #removeWords(de_stop_words + config.get("preprocessing","custom_words").split(",")),
+    removeWords(de_stop_words + config.get("preprocessing","custom_words").split(",")),
 
-    #removeENT("PERSON"),
-    #keepPOS(["NOUN"]),
     #keepUniqeTokens(),
     #keepENT(config.get("preprocessing","ents2keep"))
 

testo.py

@@ -0,0 +1,419 @@
+# -*- coding: utf-8 -*-
+
+
+import time
+start = time.time()
+
+import logging
+
+import csv
+import functools
+import os.path
+import re
+import subprocess
+import time
+import xml.etree.ElementTree as ET
+import sys
+import spacy
+import textacy
+from scipy import *
+from textacy import Vectorizer
+import warnings
+import configparser as ConfigParser
+import sys
+
+
+csv.field_size_limit(sys.maxsize)
+
+
+
+
+# Load the configuration file
+config_ini = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/config.ini"
+
+config = ConfigParser.ConfigParser()
+with open(config_ini) as f:
+    config.read_file(f)
+
+
+
+
+# config logging
+logging.basicConfig(filename=config.get("filepath","logfile"), level=logging.INFO)
+
+
+
+thesauruspath = config.get("filepath","thesauruspath")
+THESAURUS = list(textacy.fileio.read_csv(thesauruspath, delimiter=";"))
+
+
+DE_PARSER = spacy.load("de") #todo spacherkennung idee: verschiedene Corpi für verschiedene Sprachen
+de_stop_words=list(__import__("spacy." + DE_PARSER.lang, globals(), locals(), ['object']).STOP_WORDS)
+
+
+LEMMAS=config.get("filepath","lemmas")
+
+VORNAMEN = list(textacy.fileio.read_file_lines("vornamen.txt"))
+
+
+regex_specialChars = r'[`\-=~!#@,.$%^&*()_+\[\]{};\'\\:"|</>?]'
+regex_topLvl = r'\.[a-z]{2,3}(\.[a-z]{2,3})?'
+
+
+mentionFinder = re.compile(r"@[a-z0-9_]{1,15}", re.IGNORECASE)
+emailFinder = re.compile(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.IGNORECASE)
+urlFinder = re.compile(r"^(?:https?:\/\/)?(?:www\.)?[a-zA-Z0-9./]+$", re.IGNORECASE)
+topLVLFinder = re.compile(r'\.[a-z]{2,3}(\.[a-z]{2,3})?', re.IGNORECASE)
+specialFinder = re.compile(r'[`\-=~!@#$%^&*()_+\[\]{};\'\\:"|<,./>?]', re.IGNORECASE)
+hardSFinder = re.compile(r'[ß]', re.IGNORECASE)
+
+
+def printlog(string, level="INFO"):
+    """log and prints"""
+    print(string)
+    if level=="INFO":
+        logging.info(string)
+    elif level=="DEBUG":
+        logging.debug(string)
+    elif level == "WARNING":
+        logging.warning(string)
+printlog("Load functions")
+
+def compose(*functions):
+    def compose2(f, g):
+        return lambda x: f(g(x))
+    return functools.reduce(compose2, functions, lambda x: x)
+
+def get_calling_function():
+    """finds the calling function in many decent cases.
+    https://stackoverflow.com/questions/39078467/python-how-to-get-the-calling-function-not-just-its-name
+    """
+    fr = sys._getframe(1)   # inspect.stack()[1][0]
+    co = fr.f_code
+    for get in (
+        lambda:fr.f_globals[co.co_name],
+        lambda:getattr(fr.f_locals['self'], co.co_name),
+        lambda:getattr(fr.f_locals['cls'], co.co_name),
+        lambda:fr.f_back.f_locals[co.co_name], # nested
+        lambda:fr.f_back.f_locals['func'],  # decorators
+        lambda:fr.f_back.f_locals['meth'],
+        lambda:fr.f_back.f_locals['f'],
+        ):
+        try:
+            func = get()
+        except (KeyError, AttributeError):
+            pass
+        else:
+            if func.__code__ == co:
+                return func
+    raise AttributeError("func not found")
+
+
+def printRandomDoc(textacyCorpus):
+    import random
+    print()
+
+    printlog("len(textacyCorpus) = %i" % len(textacyCorpus))
+    randIndex = int((len(textacyCorpus) - 1) * random.random())
+    printlog("Index: {0} ; Text: {1} ; Metadata: {2}".format(randIndex, textacyCorpus[randIndex].text, textacyCorpus[randIndex].metadata))
+
+    print()
+
+
+
+def csv_to_contentStream(path2csv: str, content_collumn_name: str):
+    """
+    :param path2csv: string
+    :param content_collumn_name: string
+    :return: string-generator
+    """
+    stream = textacy.fileio.read_csv(path2csv, delimiter=";")  # ,encoding='utf8')
+    content_collumn = 0  # standardvalue
+
+    for i,lst in enumerate(stream):
+        if i == 0:
+            # look for desired column
+            for j,col in enumerate(lst):
+                if col == content_collumn_name:
+                    content_collumn = j
+        else:
+            yield lst[content_collumn]
+
+
+
+#############  return bool
+
+def keepPOS(pos_list):
+    return lambda tok : tok.pos_ in pos_list
+
+
+def removePOS(pos_list):
+    return lambda tok : tok.pos_ not in pos_list
+
+
+def removeWords(words, keep=None):
+    if hasattr(keep, '__iter__'):
+        for k in keep:
+            try:
+                words.remove(k)
+            except ValueError:
+                pass
+
+    return lambda tok :  tok.lower_ not in words
+
+
+def keepENT(ent_list):
+    return lambda tok : tok.ent_type_ in ent_list
+
+
+def removeENT(ent_list):
+    return lambda tok: tok.ent_type_ not in ent_list
+
+
+def remove_words_containing_Numbers():
+    return lambda tok: not bool(re.search('\d', tok.lower_))
+
+
+def remove_words_containing_specialCharacters():
+    return lambda tok: not bool(re.search(regex_specialChars, tok.lower_))
+
+
+def remove_long_words():
+    return lambda tok: not len(tok.lower_) < 2
+
+def remove_short_words():
+    return lambda tok: not len(tok.lower_) > 35
+
+def remove_first_names():
+    return lambda tok: tok.lower_ not in [name.lower() for name in VORNAMEN]
+
+############# strings
+
+def stringcleaning(stringstream, funclist):
+    for string in stringstream:
+        for f in funclist:
+            string = f(string)
+        yield string
+
+
+def remove_words_containing_topLVL():
+    return lambda string: " ".join([w.lower() for w in string.split() if not re.search(regex_topLvl, w) ])
+
+
+def replaceSpecialChars(replace_with=" "):
+    return lambda string: re.sub(regex_specialChars, replace_with, string.lower())
+
+
+def replaceNumbers(replace_with="NUMBER"):
+    return lambda string : textacy.preprocess.replace_numbers(string.lower(), replace_with=replace_with)
+
+
+def replacePhonenumbers(replace_with="PHONENUMBER"):
+    return lambda string: textacy.preprocess.replace_phone_numbers(string.lower(), replace_with=replace_with)
+
+
+def replaceHardS(replace_with="ss"):
+    return lambda string: re.sub(r'[ß]',replace_with,string.lower())
+
+
+def fixUnicode():
+    return lambda string: textacy.preprocess.fix_bad_unicode(string.lower(), normalization=u'NFC')
+
+
+def lemmatizeWord(word,filepath=LEMMAS):
+    """http://www.lexiconista.com/datasets/lemmatization/"""
+    for line in list(textacy.fileio.read_file_lines(filepath=filepath)):
+        if word.lower() == line.split()[1].strip().lower():
+            return line.split()[0].strip().lower()
+    return word.lower() # falls nix gefunden wurde
+
+
+def lemmatize():
+    #todo https://alpha.spacy.io/docs/usage/adding-languages#lemmatizer
+    return lambda tok: lemmatizeWord(tok.lower_)
+
+
+def processTextstream(textstream, string_funclist, tok_funclist,parser=DE_PARSER):
+    """
+    :param textstream: string-gen
+    :param funclist: [func]
+    :param parser: spacy-parser
+    :return: string-gen
+    """
+    pipe = parser.pipe(stringcleaning(textstream,string_funclist))
+
+    for doc in pipe:
+
+        tokens = [tok for tok in doc]
+
+        tokens = processTokens(tokens,tok_funclist,parser)
+
+        yield " ".join([tok.lower_ for tok in tokens])
+
+
+def processTokens(tokens, funclist, parser):
+    # in:tokenlist, funclist
+    # out: tokenlist
+    for f in funclist:
+
+        tokens = list(filter(f, tokens))
+
+    return tokens
+
+
+string_comp=[
+    replaceHardS(),
+    remove_words_containing_topLVL(),
+    replaceSpecialChars(),
+]
+
+tok_comp=[
+    removeENT(["PERSON"]),
+    remove_words_containing_Numbers(),
+    #keepPOS(["NOUN"]),
+    removePOS(["PUNCT","SPACE","NUM"]),
+    removeWords(de_stop_words),
+
+    remove_long_words(),
+    remove_short_words(),
+
+    remove_first_names()
+]
+
+
+
+"""
+pipe=[
+
+    ##String
+    
+    fixUnicode(),
+    replaceHardS(),
+    resolveAbbrivations(),
+    
+    remove_words_containing_topLVL(),
+    
+    replaceSpecialChars(" "), (mit Leerzeichen erstzen, dadruch werden Terme wie 8203;verfügung getrennt
+    
+    remove_words_containing_Numbers(),
+
+
+
+    ##spacyParse
+    
+    removeENT("PERSON"),
+    keepPOS(["NOUN"]),
+
+    #ODER
+        
+    lemmatize(),
+    removeWords(de_stop_words + config.get("preprocessing","custom_words").split(",")),
+
+
+    # evtl.
+    spellCorrection(),
+    keepUniqeTokens(),
+
+]
+
+
+
+
+"""
+
+
+
+
+path2csv = "M42-Export/Tickets_med.csv"
+
+ticketcorpus = textacy.Corpus(DE_PARSER)
+
+
+## add files to textacy-corpus,
+printlog("add texts to textacy-corpus")
+ticketcorpus.add_texts(
+    processTextstream(csv_to_contentStream(path2csv,"Description"), string_funclist=string_comp,tok_funclist=tok_comp)
+)
+
+for i in range(10):
+    printRandomDoc(ticketcorpus)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+words = [
+    "uniaccount",
+    "nr54065467",
+    "nr54065467",
+    "455a33c5,"
+    "tvt?=",
+    "tanja.saborowski@tu-dortmund.de",
+    "-",
+    "m-sw1-vl4053.itmc.tu-dortmund.de",
+    "------problem--------"
+]
+
+topLVLFinder = re.compile(r'\.[a-z]{2,3}(\.[a-z]{2,3})?', re.IGNORECASE)
+specialFinder = re.compile(r'[`\-=~!@#$%^&*()_+\[\]{};\'\\:"|<,./>?]', re.IGNORECASE)
+
+for w in words:
+    print(stringcleaning(w,string_comp))
+    #print(bool(re.search(r'\.[a-z]{2,3}(\.[a-z]{2,3})?',w)))
+    #print(bool(re.search(r'[`\-=~!@#$%^&*()_+\[\]{};\'\\:"|<,./>?]',w)))
+    #result = specialFinder.sub(" ", w)
+    #print(re.sub(r'[`\-=~!@#$%^&*()_+\[\]{};\'\\:"|<,./>?]'," ",w))
+
+    #print(re.sub(r'\.[a-z]{2,3}(\.[a-z]{2,3})?', " ", w))
+
+"""
+
+spracherkennung
+alles nach grüße ist irrelevant außer PS:
+
+vllt kategorien in unterkategorien aufteilen 
+
+allg: 
+utf-korregieren, 
+
+emails, urls, nummern raus 
+vllt sogar alles, was ebend jenes enthält (oder auf .toplvldomain bzw. sonderzeichen enthält oder alles was ein @ enthält
+
+sinnvoller wörter von müll trennen: 8203;verfügung -> bei sonderzeichen wörter trennen
+
+abkürzungen raus: m.a, o.ä.     
+
+wörter korrigieren
+
+sinnlose bsp: nr54065467  455a33c5   tvt?=      ------problem--------
+
+"""
+
+
+
+
+