# -*- coding: utf-8 -*-


import time
start = time.time()

import logging

import csv
import functools
import os.path
import re
import subprocess
import time
import xml.etree.ElementTree as ET
import sys
import spacy
import textacy
from scipy import *
from textacy import Vectorizer
import warnings
import configparser as ConfigParser
import sys


csv.field_size_limit(sys.maxsize)




# Load the configuration file
config_ini = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/config.ini"

config = ConfigParser.ConfigParser()
with open(config_ini) as f:
    config.read_file(f)




# config logging
logging.basicConfig(filename=config.get("filepath","logfile"), level=logging.INFO)



thesauruspath = config.get("filepath","thesauruspath")
THESAURUS = list(textacy.fileio.read_csv(thesauruspath, delimiter=";"))


DE_PARSER = spacy.load("de") #todo spacherkennung idee: verschiedene Corpi für verschiedene Sprachen
de_stop_words=list(__import__("spacy." + DE_PARSER.lang, globals(), locals(), ['object']).STOP_WORDS)


LEMMAS=config.get("filepath","lemmas")

VORNAMEN = list(textacy.fileio.read_file_lines("vornamen.txt"))


regex_specialChars = r'[`\-=~!#@,.$%^&*()_+\[\]{};\'\\:"|</>?]'
regex_topLvl = r'\.[a-z]{2,3}(\.[a-z]{2,3})?'


mentionFinder = re.compile(r"@[a-z0-9_]{1,15}", re.IGNORECASE)
emailFinder = re.compile(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.IGNORECASE)
urlFinder = re.compile(r"^(?:https?:\/\/)?(?:www\.)?[a-zA-Z0-9./]+$", re.IGNORECASE)
topLVLFinder = re.compile(r'\.[a-z]{2,3}(\.[a-z]{2,3})?', re.IGNORECASE)
specialFinder = re.compile(r'[`\-=~!@#$%^&*()_+\[\]{};\'\\:"|<,./>?]', re.IGNORECASE)
hardSFinder = re.compile(r'[ß]', re.IGNORECASE)


def printlog(string, level="INFO"):
    """log and prints"""
    print(string)
    if level=="INFO":
        logging.info(string)
    elif level=="DEBUG":
        logging.debug(string)
    elif level == "WARNING":
        logging.warning(string)
printlog("Load functions")

def compose(*functions):
    def compose2(f, g):
        return lambda x: f(g(x))
    return functools.reduce(compose2, functions, lambda x: x)

def get_calling_function():
    """finds the calling function in many decent cases.
    https://stackoverflow.com/questions/39078467/python-how-to-get-the-calling-function-not-just-its-name
    """
    fr = sys._getframe(1)   # inspect.stack()[1][0]
    co = fr.f_code
    for get in (
        lambda:fr.f_globals[co.co_name],
        lambda:getattr(fr.f_locals['self'], co.co_name),
        lambda:getattr(fr.f_locals['cls'], co.co_name),
        lambda:fr.f_back.f_locals[co.co_name], # nested
        lambda:fr.f_back.f_locals['func'],  # decorators
        lambda:fr.f_back.f_locals['meth'],
        lambda:fr.f_back.f_locals['f'],
        ):
        try:
            func = get()
        except (KeyError, AttributeError):
            pass
        else:
            if func.__code__ == co:
                return func
    raise AttributeError("func not found")


def printRandomDoc(textacyCorpus):
    import random
    print()

    printlog("len(textacyCorpus) = %i" % len(textacyCorpus))
    randIndex = int((len(textacyCorpus) - 1) * random.random())
    printlog("Index: {0} ; Text: {1} ; Metadata: {2}".format(randIndex, textacyCorpus[randIndex].text, textacyCorpus[randIndex].metadata))

    print()



def csv_to_contentStream(path2csv: str, content_collumn_name: str):
    """
    :param path2csv: string
    :param content_collumn_name: string
    :return: string-generator
    """
    stream = textacy.fileio.read_csv(path2csv, delimiter=";")  # ,encoding='utf8')
    content_collumn = 0  # standardvalue

    for i,lst in enumerate(stream):
        if i == 0:
            # look for desired column
            for j,col in enumerate(lst):
                if col == content_collumn_name:
                    content_collumn = j
        else:
            yield lst[content_collumn]



#############  return bool

def keepPOS(pos_list):
    return lambda tok : tok.pos_ in pos_list


def removePOS(pos_list):
    return lambda tok : tok.pos_ not in pos_list


def removeWords(words, keep=None):
    if hasattr(keep, '__iter__'):
        for k in keep:
            try:
                words.remove(k)
            except ValueError:
                pass

    return lambda tok :  tok.lower_ not in words


def keepENT(ent_list):
    return lambda tok : tok.ent_type_ in ent_list


def removeENT(ent_list):
    return lambda tok: tok.ent_type_ not in ent_list


def remove_words_containing_Numbers():
    return lambda tok: not bool(re.search('\d', tok.lower_))


def remove_words_containing_specialCharacters():
    return lambda tok: not bool(re.search(regex_specialChars, tok.lower_))


def remove_long_words():
    return lambda tok: not len(tok.lower_) < 2

def remove_short_words():
    return lambda tok: not len(tok.lower_) > 35

def remove_first_names():
    return lambda tok: tok.lower_ not in [name.lower() for name in VORNAMEN]



#falls wort nicht in vocab, erst schauen ob teilwort bekannt ist, falls ja, schauen ob es davor oder danach bullshit stehen hat. ggf trennen




############# strings

def stringcleaning(stringstream, funclist):
    for string in stringstream:
        for f in funclist:
            string = f(string)
        yield string


def seperate_words_on_regex(regex=regex_specialChars):
    return lambda string: " ".join(re.compile(regex).split(string))


def remove_words_containing_topLVL():
    return lambda string: " ".join([w.lower() for w in string.split() if not re.search(regex_topLvl, w) ])


def replaceSpecialChars(replace_with=" "):
    return lambda string: re.sub(regex_specialChars, replace_with, string.lower())


def replaceNumbers(replace_with="NUMBER"):
    return lambda string : textacy.preprocess.replace_numbers(string.lower(), replace_with=replace_with)


def replacePhonenumbers(replace_with="PHONENUMBER"):
    return lambda string: textacy.preprocess.replace_phone_numbers(string.lower(), replace_with=replace_with)


def replaceSharpS(replace_with="ss"):
    return lambda string: re.sub(r'[ß]',replace_with,string.lower())


def replaceRockDots():
    return lambda string: re.sub(r'[ß]', "ss", (re.sub(r'[ö]', "oe", (re.sub(r'[ü]', "ue", (re.sub(r'[ä]', "ae", string.lower())))))))



def fixUnicode():
    return lambda string: textacy.preprocess.fix_bad_unicode(string.lower(), normalization=u'NFC')


def lemmatizeWord(word,filepath=LEMMAS):
    """http://www.lexiconista.com/datasets/lemmatization/"""
    for line in list(textacy.fileio.read_file_lines(filepath=filepath)):
        if word.lower() == line.split()[1].strip().lower():
            return line.split()[0].strip().lower()
    return word.lower() # falls nix gefunden wurde


def lemmatize():
    #todo https://alpha.spacy.io/docs/usage/adding-languages#lemmatizer
    return lambda doc: " ".join([lemmatizeWord(tok.lower_) for tok in doc])


def processTextstream(textstream, string_funclist, tok_funclist, parser=DE_PARSER, single_doc_func=None):
    """
    :param textstream: string-gen
    :param funclist: [func]
    :param parser: spacy-parser
    :return: string-gen
    """
    #zuerst die string-methoden
    pipe = parser.pipe(stringcleaning(textstream,string_funclist))
    tokens=[]
    for doc in pipe:

        tokens = [tok for tok in doc]

        #dann die auf tokens
        tokens = processTokens(tokens,tok_funclist)

        if single_doc_func is not None:
            yield single_doc_func(parser(" ".join([tok.lower_ for tok in tokens])))
        else:
            yield " ".join([tok.lower_ for tok in tokens])





def processTokens(tokens, funclist):
    # in:tokenlist, funclist
    # out: tokenlist
    for f in funclist:

        tokens = list(filter(f, tokens))

    return tokens




string_comp=[
    fixUnicode(),
    replaceRockDots(),
    remove_words_containing_topLVL(),
    seperate_words_on_regex()
]

tok_comp=[
    #removeENT(["PERSON"]),
    remove_words_containing_Numbers(),
    removePOS(["PUNCT","SPACE","NUM"]),
    removeWords(de_stop_words),

    remove_long_words(),
    remove_short_words(),

    remove_first_names(),

    #keepPOS(["NOUN"]),

]

single_doc_func = lemmatize()




"""
pipe=[

    ##String
    
    fixUnicode(),
    replaceHardS(),
    resolveAbbrivations(),
    
    remove_words_containing_topLVL(),
    
    replaceSpecialChars(" "), (mit Leerzeichen erstzen, dadruch werden Terme wie 8203;verfügung getrennt
    
    remove_words_containing_Numbers(),



    ##spacyParse
    
    removeENT("PERSON"),
    keepPOS(["NOUN"]),

    #ODER
        
    lemmatize(),
    removeWords(de_stop_words + config.get("preprocessing","custom_words").split(",")),


    # evtl.
    spellCorrection(),
    keepUniqeTokens(),

]




"""




path2csv = "M42-Export/Tickets_med.csv"

ticketcorpus = textacy.Corpus(DE_PARSER)


## add files to textacy-corpus,
printlog("add texts to textacy-corpus")
ticketcorpus.add_texts(
    processTextstream(csv_to_contentStream(path2csv,"Description"), string_funclist=string_comp, tok_funclist=tok_comp, single_doc_func=single_doc_func)
)

for i in range(10):
    printRandomDoc(ticketcorpus)


























"""

spracherkennung
alles nach grüße ist irrelevant außer PS:

vllt kategorien in unterkategorien aufteilen 

allg: 
utf-korregieren, 

emails, urls, nummern raus 
vllt sogar alles, was ebend jenes enthält (oder auf .toplvldomain bzw. sonderzeichen enthält oder alles was ein @ enthält

sinnvoller wörter von müll trennen: 8203;verfügung -> bei sonderzeichen wörter trennen

abkürzungen raus: m.a, o.ä.     

wörter korrigieren

sinnlose bsp: nr54065467  455a33c5   tvt?=      ------problem--------

"""