From b542c4285a8772dd160f7f88a44bb3ad987a9651 Mon Sep 17 00:00:00 2001
From: "jannis.grundmann" <jannis.grundmann@tu-dortmund.de>
Date: Mon, 9 Oct 2017 12:50:34 +0200
Subject: [PATCH] eigene corpus-save/load methoden geschreiben

---
 java_LabledLDA/models/tickets/tickets.gz | Bin 433 -> 430 bytes
 testo.py                                 |  22 +++--
 testra.py                                | 116 ++++++++++++++++++++++-
 3 files changed, 127 insertions(+), 11 deletions(-)

diff --git a/java_LabledLDA/models/tickets/tickets.gz b/java_LabledLDA/models/tickets/tickets.gz
index fd68e5b8609364d71418660ea3610eb513da7f6f..e21f98882b49a4a5bea41d7571189bef1afa52a9 100644
GIT binary patch
literal 430
zcmV;f0a5-RiwFquEz(&6|8!|%Yh`qE0JKs|uG=sS+~*WLLcga8yr5&+7A(;MMJ2B}
zeMio7@|pnc+jL==5;&aUjE-MAV(TUN;rO*9mnh?^#OlUV!7+**j-O~E32Naqd)0{s
zxr!+}aqkjhyLwbWCsC(KdX_o}Wj==(F%To##GRn8r<ueqRkUC*_)0c8&6&*=jL&DS
z4qEbGnK(K1ps}Sf?t=F}<k5M>LWY&L29t3V{q%MV|GC^Y31W-aA=E@sW)pqg^yL3-
zM{f97OQDh9CNUN#tE1MGT7us?+K3g`W7WmOszI8_`0l}3tPeW0^4^K>dI7;!JWeTi
z+Z<qk#wt?-fwnpLy`xfdUJc@Gy%dO@*JdsaAxBCKnL$N+uA5D;Y>qcD@nBL)b9OJP
zy4Yqs%3grw0lJpe(2!RhM6a-Ki7eJcZ2jo6FajaSrD=`#N5F5Ss|ytY)Q}hZEuw_E
z`wBlEkD#V&ot~b;WSr8!p8HGm_nwczKc{;e{g=!idH0wHenova{_H4n_BGR%e1!va
Y_vBUG;Wq#P0RR6303`zSg!Tgf02I#C=l}o!

literal 433
zcmV;i0Z#rOiwFobZOd5#|8!|%Yh`qE0JKs|j@vK{-RBfMLcga8yr5#*7A#Q$MJCgn
zzLYagrU}r_ri&ny!S~@Kk6#8r2^95k{5pVAqNy+-tKi)HnMe-DPc#8VmUvp!c!rU#
zB#a6elw#_i9+jgrvBFFN$wq>T=a`ZcIz?yP1%jOxAh{TEb7I$PRp_)-6lGU@ezU2|
zg8r(2Ss}QLtX*+8y#KpBI2RvCe59>GXqt#WJ>80b4!1G6*`_u|%aCXh;@3$}`rl^6
zj!)JaO!zjFASsAVY&rK0ej8vDX8$}HuNn(S>L$~B24|8GdF<uA(|^~C3v9*Xm815_
z9qgdVXzn1F?FoJ#pjoX&+?<q4bF)k7;ldbef{bo6GI2oLV)DqAbOQs9Lap^=_rh37
zw&1-C<X9Zt*IL9IYT=Ri>hY}!NxF%x-@7DEjv(Jl_nPiUfN#J>38F$HM++1aUmNOe
zQ`zbFNq#(T!SdDSr;C^cbN<)Wzg)n*=i~mLmwcoC%jzF%56<WM?DgUJbAW_ZONH$K
b%!7b?G_URszX1RM|NjF3^`h4i{R0316rtBk

diff --git a/testo.py b/testo.py
index b3ff86c..46849ef 100644
--- a/testo.py
+++ b/testo.py
@@ -7,8 +7,8 @@ print(datetime.now())
 
 
 #path2csv = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/M42-Export/Tickets_med.csv"
-path2csv = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/M42-Export/Tickets_small.csv"
-#path2csv = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/M42-Export/de_tickets.csv"
+#path2csv = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/M42-Export/Tickets_small.csv"
+path2csv = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/M42-Export/de_tickets.csv"
 
 path_csv_split = path2csv.split("/")
 print(path_csv_split[len(path_csv_split)-1])
@@ -501,7 +501,7 @@ def build_thesaurus(path2lexicalentries, path2synsets):
 
 
 THESAURUS=[]
-#THESAURUS=build_thesaurus(path2lexicalentries=lexicalentries,path2synsets=synsets) #todo anschalten
+THESAURUS=build_thesaurus(path2lexicalentries=lexicalentries,path2synsets=synsets)
 
 def getFirstSynonym(word, thesaurus=THESAURUS):
     if not isinstance(word, str):
@@ -541,7 +541,7 @@ def words(text): return re.findall(r'\w+', text.lower())
 
 
 WORDS={}
-#WORDS = Counter(words(open(path2words).read())) #todo anschalten
+WORDS = Counter(words(open(path2words).read()))
 
 def P(word, N=sum(WORDS.values())):
     "Probability of `word`."
@@ -629,10 +629,10 @@ def stringcleaning(stringstream):
         string = " ".join([lemmatizeWord(word) for word in string.split()])
 
         # synonyme normalisieren    #idee vor oder nach lemmatize?
-        #string = " ".join([getFirstSynonym(word) for word in string.split()])
+        string = " ".join([getFirstSynonym(word) for word in string.split()])
 
         # autocorrect
-        #string = " ".join([autocorrectWord(word) for word in string.split()])
+        string = " ".join([autocorrectWord(word) for word in string.split()])
 
         yield string
 
@@ -745,7 +745,7 @@ custom_words=["geehrt","dame","herr","hilfe","problem","lauten","bedanken","vora
               "auffahren","vorgang","hinweis","institut","universitaet","name","gruss","id","erfolg","mail","folge",
               "nummer","team","fakultaet","email","absender","tu","versenden","vorname","message",
               "service","strasse","prozess","portal","raum","personal","moeglichkeit","fremd","wende","rueckfrage", "stehen", "verfuegung"
-              "funktionieren","kollege", "pruefen"
+              "funktionieren","kollege", "pruefen","hoffen"
               ]
 
 
@@ -896,9 +896,15 @@ weighting = 'tf'
 # weighting ='tfidf'
 named_entities = False
 
-#printvecotorization(ngrams=ngrams,min_df=min_df,max_df=max_df,weighting=weighting,named_entities=named_entities)
+printvecotorization(ngrams=1,min_df=1,max_df=1.0,weighting=weighting)
+printvecotorization(ngrams=1,min_df=1,max_df=0.5,weighting=weighting)
+printvecotorization(ngrams=1,min_df=1,max_df=0.8,weighting=weighting)
 
 
+printvecotorization(ngrams=(1,2),min_df=1,max_df=1.0,weighting=weighting)
+printvecotorization(ngrams=(1,2),min_df=1,max_df=0.5,weighting=weighting)
+printvecotorization(ngrams=(1,2),min_df=1,max_df=0.8,weighting=weighting)
+
 
 
 """
diff --git a/testra.py b/testra.py
index 0be8eac..5b81579 100644
--- a/testra.py
+++ b/testra.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import re
 import time
+import json
 
 import spacy
 import textacy
@@ -16,7 +17,116 @@ import xml.etree.ElementTree as ET
 print(datetime.now())
 
 
-#PARSER=spacy.load("de")
+PARSER=spacy.load("de")
+
+
+corpus = textacy.Corpus(PARSER)
+
+testcontetn = [
+    "fdsfdsfsd",
+    "juzdtjlkö",
+    "gfadojplk"
+]
+
+testmetda = [
+    {"categoryName":"zhb","Solution":"","Subject":"schulungstest"},
+    {"categoryName":"neuanschluss","Solution":"subject","Subject":"telephone contract"},
+    {"categoryName":"zhb","Solution":"","Subject":"setuji"}
+]
+
+
+def makecontent(testcontetn):
+    for content in testcontetn:
+        yield content
+
+
+def makemeta( testmetda):
+    for  metdata in  testmetda:
+        yield  metdata
+
+
+corpus.add_texts(
+    makecontent(testcontetn),
+    makemeta(testmetda)
+)
+
+print(corpus)
+
+
+corpus_path = "/home/jannis.grundmann/PycharmProjects/topicModelingTickets/corpus/"
+corpus_name = "testcorpus"
+
+"""
+#corpus.save(corpus_path, name=corpus_name, compression=corpus_compression)
+#corpus =  textacy.Corpus.load(corpus_path, name=corpus_name, compression=corpus_compression)
+
+
+
+import pathlib
+
+strings_path = pathlib.Path(corpus_path + 'strings.json')
+path_lexemes_bin_ = pathlib.Path(corpus_path + 'lexemes.bin')
+
+PARSER.vocab.dump(path_lexemes_bin_)
+nlp.vocab.load_lexemes(path_lexemes_bin_)
+"""
+
+def save_corpus(corpus_path,corpus_name):
+
+    # save stringstore
+    stringstore_path = corpus_path + corpus_name + '_strings.json'
+    with open(stringstore_path, "w") as file:
+        PARSER.vocab.strings.dump(file)
+
+
+    #save content
+    contentpath = corpus_path + corpus_name+ "_content.bin"
+    textacy.fileio.write_spacy_docs((doc.spacy_doc for doc in corpus),contentpath)
+
+
+    #save meta
+    metapath = corpus_path + corpus_name +"_meta.json"
+    textacy.fileio.write_json_lines((doc.metadata for doc in corpus), metapath)
+
+
+
+def load_corpus(corpus_path,corpus_name):
+    # load new lang
+    nlp = spacy.load("de")
+
+    #load stringstore
+    stringstore_path = corpus_path + corpus_name + '_strings.json'
+    with open(stringstore_path,"r") as file:
+        nlp.vocab.strings.load(file)
+
+    # define corpus
+    corpus = textacy.Corpus(nlp)
+
+    # load meta
+    metapath = corpus_path + corpus_name +"_meta.json"
+    metadata_stream = textacy.fileio.read_json_lines(metapath)
+
+    #load content
+    contentpath = corpus_path + corpus_name+ "_content.bin"
+    spacy_docs = textacy.fileio.read_spacy_docs(corpus.spacy_vocab, contentpath)
+
+    for spacy_doc, metadata in zip(spacy_docs, metadata_stream):
+        corpus.add_doc(
+            textacy.Doc(spacy_doc, lang=corpus.spacy_lang, metadata=metadata))
+
+    return corpus
+
+
+save_corpus(corpus_path,corpus_name)
+
+print(load_corpus(corpus_path,corpus_name))
+
+
+
+
+
+#idee das auch mit spellchecker, lemmetaizer und thesaurus machen wegen memory
+# todo generators immer neu laden? wegen laufzeit-nacheinander-picking, denn sonst nicht det
 
 
 """
@@ -97,7 +207,7 @@ for r in root:
 
 
 
-
+"""
 import re
 from collections import Counter
 
@@ -135,7 +245,7 @@ def edits2(word):
     "All edits that are two edits away from `word`."
     return (e2 for e1 in edits1(word) for e2 in edits1(e1))
 
-
+"""