1 # -*- coding: utf-8 -*-
2 #Author: Pierre Ratinaud
3 #Copyright (c) 2008-2020 Pierre Ratinaud
4 #modification pour python 3 : Laurent Mérat, 6x7 - mai 2020
7 #------------------------------------
8 # import des modules python
9 #------------------------------------
19 from operator import itemgetter
20 from uuid import uuid4
23 #------test spacy------------
25 #nlp = spacy.load("fr_core_news_lg")
27 #------------------------------------
28 # import des fichiers du projet
29 #------------------------------------
30 from functions import decoupercharact, ReadDicoAsDico, DoConf, ReadLexique, progressbar
31 from chemins import PathOut
32 from dialog import CorpusPref, SubTextFromMetaDial, MergeClusterFrame
33 from colors import colors
39 log = logging.getLogger('iramuteq.corpus')
42 def copycorpus(corpus) :
43 log.info('copy corpus')
44 copy_corpus = Corpus(corpus.parent, parametres = corpus.parametres)
45 copy_corpus.ucis = corpus.ucis
46 copy_corpus.formes = corpus.formes
47 copy_corpus.pathout = corpus.pathout
48 copy_corpus.conn_all()
52 return Uce(uce.ident, uce.para, uce.uci)
55 nuci = Uci(uci.ident, '')
56 nuci.etoiles = copy(uci.etoiles)
57 nuci.uces = [CopyUce(uce) for uce in uci.uces]
58 nuci.paras = copy(uci.paras)
67 def __init__(self, parent, parametres = {}, read = False) :
69 self.parametres = parametres
71 self.connformes = None
73 self.conncorpus = None
80 self.idformesuces = {}
85 self.pathout = PathOut(dirout = parametres['pathout'])
88 def add_word(self, word) :
89 if word in self.formes :
90 self.formes[word].freq += 1
91 if self.formes[word].ident in self.idformesuces :
92 if self.ucis[-1].uces[-1].ident in self.idformesuces[self.formes[word].ident] :
93 self.idformesuces[self.formes[word].ident][self.ucis[-1].uces[-1].ident] += 1
95 self.idformesuces[self.formes[word].ident][self.ucis[-1].uces[-1].ident] = 1
97 self.idformesuces[self.formes[word].ident] = {self.ucis[-1].uces[-1].ident: 1}
99 if word in self.parent.lexique :
100 gramtype = self.parent.lexique[word][1]
101 lem = self.parent.lexique[word][0]
102 elif word.isdigit() :
108 self.formes[word] = Word(word, gramtype, len(self.formes), lem)
109 self.idformesuces[self.formes[word].ident] = {self.ucis[-1].uces[-1].ident : 1}
111 def add_word_from_forme(self, word, stident):
112 if word.forme in self.formes :
113 self.formes[word.forme].freq += 1
114 if self.formes[word.forme].ident in self.idformesuces :
115 if stident in self.idformesuces[self.formes[word.forme].ident] :
116 self.idformesuces[self.formes[word.forme].ident][stident] += 1
118 self.idformesuces[self.formes[word.forme].ident][stident] = 1
120 self.idformesuces[self.formes[word.forme].ident] = {stident: 1}
122 self.formes[word.forme] = Word(word.forme, word.gram, len(self.formes), word.lem)
123 self.idformesuces[self.formes[word.forme].ident] = {stident : 1}
126 """connect corpus to db"""
127 if self.connformes is None :
128 log.info('connexion corpus')
129 self.connuces = sqlite3.connect(self.pathout['uces.db'])
130 self.cuces = self.connuces.cursor()
131 self.connformes = sqlite3.connect(self.pathout['formes.db'])
132 self.cformes = self.connformes.cursor()
133 self.conncorpus = sqlite3.connect(self.pathout['corpus.db'])
134 self.ccorpus = self.conncorpus.cursor()
135 self.cformes.execute('PRAGMA temp_store=MEMORY;')
136 self.cformes.execute('PRAGMA journal_mode=MEMORY;')
137 self.cformes.execute('PRAGMA synchronous = OFF;')
138 self.cuces.execute('PRAGMA temp_store=MEMORY;')
139 self.cuces.execute('PRAGMA journal_mode=MEMORY;')
140 self.cuces.execute('PRAGMA synchronous = OFF;')
141 self.ccorpus.execute('PRAGMA temp_store=MEMORY;')
142 self.ccorpus.execute('PRAGMA journal_mode=MEMORY;')
143 self.ccorpus.execute('PRAGMA synchronous = OFF;')
145 def read_corpus(self) :
146 log.info('read corpus')
147 self.parametres['syscoding'] = 'utf8'
148 if self.conncorpus is None :
150 res = self.ccorpus.execute('SELECT * FROM etoiles;')
152 self.ucis.append(Uci(row[0], row[1], row[2]))
153 uces = self.conncorpus.cursor().execute('SELECT * FROM luces where uci=?;',(repr(self.ucis[-1].ident),))
155 self.ucis[-1].uces.append(Uce(uce[2], uce[1], uce[0]))
156 res = self.ccorpus.execute('SELECT * FROM formes;')
157 self.formes = dict([[forme[1], Word(forme[1], forme[3], forme[0], lem = forme[2], freq = forme[4])] for forme in res])
160 def getworduces(self, wordid) :
161 if isinstance(wordid, str) :
162 wordid = self.formes[wordid].ident
163 res = self.cformes.execute('SELECT uces FROM uces where id=? ORDER BY id;', (repr(wordid),))
164 return list(itertools.chain(*[[int(val) for val in row[0].split()] if not isinstance(row[0], int) else [row[0]] for row in res]))
166 def getworducis(self, wordid) :
167 res = self.getworduces(wordid)
168 return list(set([self.getucefromid(uce).uci for uce in res]))
170 def getformeuceseff(self, formeid) :
171 if isinstance(formeid, str) :
172 formeid = self.formes[formeid].ident
173 res = self.cformes.execute('SELECT uces FROM uces where id=? ORDER BY id;', (repr(formeid),))
174 uces = list(itertools.chain(*[[int(val) for val in row[0].split()] if not isinstance(row[0], int) else [row[0]] for row in res]))
175 query = 'SELECT eff FROM eff where id=%i ORDER BY id' % formeid
176 res = self.cformes.execute(query)
177 eff = list(itertools.chain(*[[int(val) for val in row[0].split()] if not isinstance(row[0], int) else [row[0]] for row in res]))
179 for i, uce in enumerate(uces) :
180 formeuceeff[uce] = formeuceeff.get(uce, 0) + eff[i]
183 def getlemuces(self, lem) :
184 formesid = ', '.join([repr(val) for val in self.lems[lem].formes])
185 query = 'SELECT uces FROM uces where id IN (%s) ORDER BY id' % formesid
186 res = self.cformes.execute(query)
187 return list(set(list(itertools.chain(*[[int(val) for val in row[0].split()] if not isinstance(row[0], int) else [row[0]] for row in res]))))
189 def gettgenst(self, tgen):
192 if lem in self.lems :
193 formesid += self.lems[lem].formes
195 print('abscent : %s' % lem)
196 query = 'SELECT uces FROM uces where id IN %s ORDER BY id' % str(tuple(formesid))
197 res = self.cformes.execute(query)
198 return list(set(list(itertools.chain(*[[int(val) for val in row[0].split()] if not isinstance(row[0], int) else [row[0]] for row in res]))))
200 def gettgenstprof(self, tgen, classe, i, clnb):
203 if lem in self.lems :
204 lemst = self.getlemuces(lem)
206 if not lem in self.tgenlem :
207 self.tgenlem[lem] = [0] * clnb
208 self.tgenlem[lem][i] = len(set(lemst).intersection(classe))
210 print('abscent: ',lem)
211 return list(set(tgenst))
213 def gettgentxt(self, tgen):
214 sts = self.gettgenst(tgen)
215 return list(set([self.getucefromid(val).uci for val in sts]))
217 def getlemucis(self, lem) :
218 uces = self.getlemuces(lem)
219 return list(set([self.getucefromid(val).uci for val in uces]))
221 def getlemuceseff(self, lem, luces = None) :
222 formesid = ', '.join([repr(val) for val in self.lems[lem].formes])
223 query = 'SELECT uces FROM uces where id IN (%s) ORDER BY id' % formesid
224 res = self.cformes.execute(query)
225 uces = list(itertools.chain(*[[int(val) for val in row[0].split()] if not isinstance(row[0], int) else [row[0]] for row in res]))
226 query = 'SELECT eff FROM eff where id IN (%s) ORDER BY id' % formesid
227 res = self.cformes.execute(query)
228 eff = list(itertools.chain(*[[int(val) for val in row[0].split()] if not isinstance(row[0], int) else [row[0]] for row in res]))
230 for i, uce in enumerate(uces) :
231 lemuceeff[uce] = lemuceeff.get(uce, 0) + eff[i]
234 def getlemclustereff(self, lem, cluster) :
235 return len(list(set(self.lc[cluster]).intersection(self.getlemuces(lem))))
237 def getlemeff(self, lem) :
238 return self.lems[lem].freq
243 def getforme(self, formeid) :
244 if self.idformes is None : self.make_idformes()
245 return self.idformes[formeid]
247 def gettotocc(self) :
248 return sum([self.formes[forme].freq for forme in self.formes])
250 def getucemean(self) :
251 return float(self.gettotocc())/self.getucenb()
254 return self.ucis[-1].uces[-1].ident + 1
257 return self.ucis[-1].ident + 1
259 def getucisize(self) :
260 ucesize = self.getucesize()
261 return [sum(ucesize[uci.uces[0].ident:(uci.uces[-1].ident + 1)]) for uci in self.ucis]
263 def getucesize(self) :
264 res = self.getalluces()
265 return [len(uce[1].split()) for uce in res]
267 def getconcorde(self, uces) :
268 return self.cuces.execute('select * from uces where id IN (%s) ORDER BY id;' % ', '.join([repr(i) for i in uces]))
270 def getuciconcorde(self, ucis) :
271 uces = [[val,[uce.ident for uce in self.ucis[val].uces]] for val in ucis]
272 uces = [[val[0], '\n'.join([row[1] for row in self.getconcorde(val[1])])] for val in uces]
275 def getuciconcorde_uces(self, uciid, uceid) :
276 uces = [uce.ident for uce in self.ucis[uciid].uces]
277 uces = [row for row in self.getconcorde(uces)]
280 def getwordconcorde(self, word) :
281 return self.getconcorde(self.getworduces(word))
283 def getlemconcorde(self, lem) :
284 return self.getconcorde(self.getlemuces(lem))
286 def getalluces(self) :
287 return self.cuces.execute('SELECT * FROM uces')
289 def getallucis(self):
290 uces = [row[1] for row in self.getalluces()]
291 return [[uci.ident, '\n'.join([uces[uce.ident] for uce in uci.uces])] for uci in self.ucis]
293 def getucesfrometoile(self, etoile) :
294 return [uce.ident for uci in self.ucis for uce in uci.uces if etoile in uci.etoiles]
296 def getetoileuces(self) :
297 log.info('get uces etoiles')
300 for uci in self.ucis :
301 etoiles = uci.etoiles[1:]
303 if et in etoileuces :
304 etoileuces[et] += [uce.ident for uce in uci.uces]
306 etoileuces[et] = [uce.ident for uce in uci.uces]
308 for et in uci.paras :
309 if et in etoileuces :
310 etoileuces[et] += [uce.ident for uce in uci.uces if uce.para == idpara]
312 etoileuces[et] = [uce.ident for uce in uci.uces if uce.para == idpara]
318 def getetoileucis(self):
320 for uci in self.ucis :
321 etoiles = uci.etoiles[1:]
323 if et in etoileuces :
324 etoileuces[et] += [uci.ident]
326 etoileuces[et] = [uci.ident]
329 def getucefromid(self, uceid) :
330 if self.iduces is None : self.make_iduces()
331 return self.iduces[uceid]
333 def gethapaxnb(self) :
334 return len([None for forme in self.formes if self.formes[forme].freq == 1])
336 def getactivesnb(self, key) :
337 return len([lem for lem in self.lems if self.lems[lem].act == key])
339 # fonction inactive mais avec une incertitude concernant l'indentation sur le dernier else
340 # def make_lems(self, lem = True) :
341 # log.info('make lems')
343 # for forme in self.formes :
344 # if self.formes[forme].lem in self.lems :
345 # if self.formes[forme].ident not in self.lems[self.formes[forme].lem] :
346 # self.lems[self.formes[forme].lem][self.formes[forme].ident] = 0
348 # self.lems[self.formes[forme].lem] = {self.formes[forme].ident : 0}
350 def getetbyuceid(self, uceid) :
351 if self.uceuci is None : self.uceuci = dict([[uce.ident,uci.ident] for uci in self.ucis for uce in uci.uces])
352 return self.ucis[self.uceuci[uceid]].etoiles
354 def make_lems(self, lem = True) :
355 log.info('make lems')
358 for forme in self.formes :
359 if self.formes[forme].lem in self.lems :
360 if self.formes[forme].ident not in self.lems[self.formes[forme].lem].formes :
361 self.lems[self.formes[forme].lem].add_forme(self.formes[forme])
363 self.lems[self.formes[forme].lem] = Lem(self, self.formes[forme])
365 self.lems = dict([[forme, Lem(self, self.formes[forme])] for forme in self.formes])
367 def make_lems_from_dict(self, dictionnaire, dolem = True) :
368 log.info('make lems from dict')
370 for forme in self.formes :
371 if self.formes[forme].forme in dictionnaire :
372 lem = dictionnaire[forme][0]
373 gram = dictionnaire[forme][1]
374 elif forme.isdigit() :
380 self.formes[forme].lem = lem
381 self.formes[forme].gram = gram
383 if self.formes[forme].lem in self.lems :
384 if self.formes[forme].ident not in self.lems[self.formes[forme].lem].formes :
385 self.lems[self.formes[forme].lem].add_forme(self.formes[forme])
387 self.lems[self.formes[forme].lem] = Lem(self, self.formes[forme])
389 self.lems[forme] = Lem(self, self.formes[forme])
391 def make_idformes(self) :
392 self.idformes = dict([[self.formes[forme].ident, self.formes[forme]] for forme in self.formes])
394 def make_iduces(self) :
395 if self.iduces is None :
396 self.iduces = dict([[uce.ident, uce] for uci in self.ucis for uce in uci.uces])
398 def make_lexitable(self, mineff, etoiles, gram = 0) :
403 tokeep = [lem for lem in self.lems if self.lems[lem].freq >= mineff and self.lems[lem].act in grams]
404 etuces = [[] for et in etoiles]
405 for uci in self.ucis :
406 get = list(set(uci.etoiles).intersection(etoiles))
408 log.info('2 variables sur une ligne')
410 etuces[etoiles.index(get[0])] += [uce.ident for uce in uci.uces]
411 etuces = [set(val) for val in etuces]
414 deff = self.getlemuceseff(lem)
415 ucesk = list(deff.keys())
416 line = [lem] + [sum([deff[uce] for uce in et.intersection(ucesk)]) for et in etuces]
417 if sum(line[1:]) >= mineff :
419 tab.insert(0, [''] + etoiles)
422 def make_tgen_table(self, tgen, etoiles, tot = None):
423 lclasses = [self.getucesfrometoile(etoile) for etoile in etoiles]
424 sets = [set(cl) for cl in lclasses]
425 totoccurrences = dict([[val, 0] for val in etoiles])
427 for forme in self.formes :
428 formeuceeff = self.getformeuceseff(forme)
429 for i, classe in enumerate(lclasses) :
430 concern = sets[i].intersection(list(formeuceeff.keys()))
432 totoccurrences[etoiles[i]] += sum([formeuceeff[uce] for uce in concern])
433 #tgenoccurrences = dict([[val, 0] for val in etoiles])
436 tgenoccurrences[t] = dict([[val, 0] for val in etoiles])
438 lemuceeff = self.getlemuceseff(lem)
439 for i, classe in enumerate(lclasses) :
440 concern = sets[i].intersection(list(lemuceeff.keys()))
442 tgenoccurrences[t][etoiles[i]] += sum([lemuceeff[uce] for uce in concern])
443 return tgenoccurrences, totoccurrences
445 def make_tgen_profile(self, tgen, ucecl, uci = False) :
446 log.info('tgen/classes')
450 #FIXME : NE MARCHE PLUS CHANGER CA
451 tab = [[lem] + [len(set(self.gettgentxt(tgen[lem])).intersection(classe)) for i, classe in enumerate(ucecl)] for lem in tgen]
453 tab = [[lem] + [len(set(self.gettgenstprof(tgen[lem], classe, i, clnb)).intersection(classe)) for i, classe in enumerate(ucecl)] for lem in tgen]
454 tab = [[line[0]] + [val for val in line[1:]] for line in tab if sum(line[1:]) >= 3]
459 #while nam + `i` in tgen :
462 #last = [nam] + [`len(classe)` for classe in ucecl]
464 #line0 = ['tgen'] + ['_'.join(['cluster', `i+1`]) for i in range(len(ucecl))]
466 #with open(fileout, 'w') as f :
467 # f.write('\n'.join(['\t'.join(line) for line in tab]).encode(self.parametres['syscoding']))
469 def make_efftype_from_etoiles(self, etoiles) :
471 etuces = [[] for et in etoiles]
472 for uci in self.ucis :
473 get = list(set(uci.etoiles).intersection(etoiles))
475 return '2 variables sur la meme ligne'
477 etuces[etoiles.index(get[0])] += [uce.ident for uce in uci.uces]
478 etuces = [set(val) for val in etuces]
479 for lem in self.lems :
480 deff = self.getlemuceseff(lem)
481 ucesk = list(deff.keys())
482 gram = self.lems[lem].gram
484 dtype[gram] = [i + j for i, j in zip(dtype[gram], [sum([deff[uce] for uce in et.intersection(ucesk)]) for et in etuces])]
486 dtype[gram] = [sum([deff[uce] for uce in et.intersection(ucesk)]) for et in etuces]
487 tabout = [[gram] + dtype[gram] for gram in dtype]
488 tabout.insert(0, [''] + etoiles)
491 def make_uceactsize(self, actives) :
492 res = self.getalluces()
495 deff = self.getlemuceseff(lem)
497 ucesize[uce] = ucesize.get(uce, 0) + 1
500 def make_uc(self, actives, lim1, lim2) :
501 uceactsize = self.make_uceactsize(actives)
507 for uce in [uce for uci in self.ucis for uce in uci.uces] :
508 if uce.para == lastpara :
510 last1 += uceactsize.get(uce.ident,0)
511 uc1[-1].append(uce.ident)
513 uc1.append([uce.ident])
516 last2 += uceactsize.get(uce.ident, 0)
517 uc2[-1].append(uce.ident)
519 uc2.append([uce.ident])
522 last1 = uceactsize.get(uce.ident, 0)
523 last2 = uceactsize.get(uce.ident, 0)
525 uc1.append([uce.ident])
526 uc2.append([uce.ident])
529 def make_and_write_sparse_matrix_from_uc(self, actives, sizeuc1, sizeuc2, uc1out, uc2out, listuce1out, listuce2out) :
530 uc1, uc2 = self.make_uc(actives, sizeuc1, sizeuc2)
531 log.info('taille uc1 : %i - taille uc2 : %i' % (len(uc1), len(uc2)))
532 self.write_ucmatrix(uc1, actives, uc1out)
533 self.write_ucmatrix(uc2, actives, uc2out)
534 listuce1 = [['uce', 'uc']] + [[repr(uce), repr(i)] for i, ucl in enumerate(uc1) for uce in ucl]
535 listuce2 = [['uce', 'uc']] + [[repr(uce), repr(i)] for i, ucl in enumerate(uc2) for uce in ucl]
536 with open(listuce1out, 'w') as f :
537 f.write('\n'.join([';'.join(line) for line in listuce1]))
538 with open(listuce2out, 'w') as f :
539 f.write('\n'.join([';'.join(line) for line in listuce2]))
540 return len(uc1), len(uc2)
542 def write_ucmatrix(self, uc, actives, fileout) :
543 log.info('write uc matrix %s' % fileout)
544 uces_uc = dict([[uce, i] for i, ucl in enumerate(uc) for uce in ucl])
547 with open(fileout + '~', 'w+') as f :
548 for i, lem in enumerate(actives) :
549 for uce in self.getlemuces(lem):
550 if (uces_uc[uce], i) not in deja_la :
552 f.write(''.join([' '.join([repr(uces_uc[uce]+1),repr(i+1),repr(1)]),'\n']))
553 deja_la[(uces_uc[uce], i)] = 0
555 with open(fileout, 'w') as ffin :
556 ffin.write("%%%%MatrixMarket matrix coordinate integer general\n%i %i %i\n" % (len(uc), len(actives), nbl))
559 os.remove(fileout + '~')
562 def export_corpus(self, outf) :
563 #outf = 'export_corpus.txt'
565 res = self.getalluces()
569 with open(outf,'w', encoding='utf8') as f :
571 if self.iduces[uce[0]].uci == actuci and self.iduces[uce[0]].para == actpara :
572 f.write(uce[1] + '\n')
573 elif self.iduces[uce[0]].uci != actuci :
574 actuci = self.iduces[uce[0]].uci
575 if self.ucis[self.iduces[uce[0]].uci].paras == [] :
576 actpara = self.iduces[uce[0]].para
577 f.write('\n' + ' '.join(self.ucis[self.iduces[uce[0]].uci].etoiles) + '\n' + uce[1] + '\n')
580 actpara = self.iduces[uce[0]].para
581 f.write('\n'.join([' '.join(self.ucis[self.iduces[uce[0]].uci].etoiles), self.ucis[self.iduces[uce[0]].uci].paras[ident], uce[1]] + '\n'))
582 elif self.iduces[uce[0]].para != actpara :
583 actpara = self.iduces[uce[0]].para
585 f.write('\n'.join([self.ucis[self.iduces[uce[0]].uci].paras[ident], uce[1]]) + '\n')
587 def export_meta_table(self, outf) :
588 metas = [[repr(i)] + text.etoiles[1:] for i, text in enumerate(self.ucis)]
589 longueur_max = max([len(val) for val in metas])
590 first = ['column_%i' % i for i in range(longueur_max)]
591 metas.insert(0, first)
592 with open(outf, 'w', encoding='utf8') as f :
593 f.write('\n'.join(['\t'.join(line) for line in metas]))
595 def export_corpus_classes(self, outf, alc = True, lem = False, uci = False) :
597 for i, lc in enumerate(self.lc) :
600 for uce in self.lc0 :
603 res = self.getalluces()
606 res = self.getallucis()
607 with open(outf, 'w', encoding='utf8') as f :
611 actuci = self.iduces[uce[0]].uci
615 guce = ' '.join([self.formes[forme].lem for forme in guce.split()])
617 etline = ' '.join(self.ucis[actuci].etoiles + ['*classe_%i' % ucecl[uce[0]]])
619 etline = ' '.join(['<' + '='.join(et.split('_')) + '>' for et in self.ucis[actuci].etoiles[1:]])
620 f.write(etline + '\n')
621 f.write(guce + '\n\n')
623 def export_classe(self, outf, classe, lem = False, uci = False) :
624 sts = self.lc[classe - 1]
626 res = self.getconcorde(sts)
629 res = self.getuciconcorde(sts)
630 with open(outf, 'w', encoding='utf8') as f :
634 f.write(' '.join(self.ucis[self.iduces[uce[0]].uci].etoiles) + '\n')
636 f.write(' '.join(self.ucis[uce[0]].etoiles) + '\n')
638 guce = ' '.join([self.formes[forme].lem for forme in guce.split()])
639 f.write(guce + '\n\n')
641 def export_owledge(self, rep, classe, lem = False, uci = False) :
642 sts = self.lc[classe - 1]
644 res = self.getconcorde(sts)
647 res = self.getuciconcorde(sts)
651 outf = '.'.join([repr(ident), 'txt'])
652 outf = os.path.join(rep, outf)
654 guce = ' '.join([self.formes[forme].lem for forme in guce.split()])
655 with open(outf, 'w', encoding='utf8') as f :
656 f.write(guce) #.encode('cp1252', errors = 'replace'))
658 def export_tropes(self, fileout, classe, lem = False, uci = False) :
659 sts = self.lc[classe - 1]
661 res = self.getconcorde(sts)
664 res = self.getuciconcorde(sts)
665 with open(fileout, 'w', encoding='utf8') as f :
669 guce = ' '.join([self.formes[forme].lem for forme in guce.split()])
670 f.write(guce) #.encode('cp1252', errors = 'replace'))
673 def make_and_write_sparse_matrix_from_uces(self, actives, outfile, listuce = False) :
674 log.info('make_and_write_sparse_matrix_from_uces %s' % outfile)
676 with open(outfile + '~', 'w+') as f :
677 for i, lem in enumerate(actives) :
678 for uce in sorted(self.getlemuces(lem)) :
680 f.write(''.join([' '.join([repr(uce+1), repr(i+1),repr(1)]),'\n']))
682 with open(outfile, 'w') as ffin :
683 ffin.write("%%%%MatrixMarket matrix coordinate integer general\n%i %i %i\n" % (self.getucenb(), len(actives), nbl))
686 os.remove(outfile + '~')
688 with open(listuce, 'w') as f :
689 f.write('\n'.join(['uce;uc'] + [';'.join([repr(i),repr(i)]) for i in range(0, self.getucenb())]))
691 def make_and_write_sparse_matrix_from_uci(self, actives, outfile, listuci = False) :
692 log.info('make_and_write_sparse_matrix_from_ucis %s' % outfile)
694 with open(outfile + '~', 'w+') as f :
695 for i, lem in enumerate(actives) :
696 for uci in sorted(self.getlemucis(lem)) :
698 f.write(''.join([' '.join([repr(uci+1), repr(i+1),repr(1)]),'\n']))
700 with open(outfile, 'w') as ffin :
701 ffin.write("%%%%MatrixMarket matrix coordinate integer general\n%i %i %i\n" % (self.getucinb(), len(actives), nbl))
704 os.remove(outfile + '~')
706 with open(listuci, 'w') as f :
707 f.write('\n'.join(['uci;uc'] + [';'.join([repr(i),repr(i)]) for i in range(0, self.getucinb())]))
709 def make_and_write_sparse_matrix_from_classe(self, actives, uces, outfile) :
710 log.info('make_and_write_sparse_matrix_from_classe %s' % outfile)
712 duces = dict([[uce, i] for i, uce in enumerate(uces)])
713 with open(outfile + '~', 'w+') as f :
714 for i, lem in enumerate(actives) :
715 uces_ok = list(set(self.getlemuces(lem)).intersection(uces))
717 f.write(''.join([' '.join([repr(duces[uce]+1),repr(i+1),repr(1)]),'\n']))
719 with open(outfile, 'w') as ffin :
720 ffin.write("%%%%MatrixMarket matrix coordinate integer general\n%i %i %i\n" % (len(uces), len(actives), nbl))
723 os.remove(outfile + '~')
725 def make_table_with_classe(self, uces, list_act, uci = False) :
726 table_uce = [[0 for val in list_act] for line in range(0,len(uces))]
727 uces = dict([[uce, i] for i, uce in enumerate(uces)])
729 getlem = self.getlemucis
731 getlem = self.getlemuces
732 for i, lem in enumerate(list_act) :
733 lemuces = list(set(getlem(lem)).intersection(uces))
735 table_uce[uces[uce]][i] = 1
736 table_uce.insert(0, list_act)
739 def make_pondtable_with_classe(self, uces, list_act) :
740 table_uce = [[0 for val in list_act] for line in range(0,len(uces))]
741 uces = dict([[uce, i] for i, uce in enumerate(uces)])
742 for i, lem in enumerate(list_act) :
743 uceseff = self.getlemuceseff(lem)
744 lemuces = list(set(uceseff.keys()).intersection(uces))
746 table_uce[uces[uce]][i] = uceseff[uce]
747 table_uce.insert(0, list_act)
750 def parse_active(self, gramact, gramsup = None) :
751 log.info('parse actives')
752 for lem in self.lems :
753 if lem.startswith('_') and lem.endswith('_') :
754 self.lems[lem].act = 2
755 elif self.lems[lem].gram in gramact :
756 self.lems[lem].act = 1
757 elif gramsup is not None and self.lems[lem].gram not in gramact:
758 if self.lems[lem].gram in gramsup :
759 self.lems[lem].act = 2
761 self.lems[lem].act = 0
763 self.lems[lem].act = 2
765 def make_actives_limit(self, limit, key = 1) :
766 if self.idformes is None :
768 return [lem for lem in self.lems if self.getlemeff(lem) >= limit and self.lems[lem].act == key]
770 def make_actives_nb(self, nbmax, key) :
771 log.info('make_actives_nb : %i - %i' % (nbmax,key))
772 if self.idformes is None :
774 allactives = [[self.lems[lem].freq, lem] for lem in self.lems if self.lems[lem].act == key and self.lems[lem].freq >= 3]
775 self.activenb = len(allactives)
776 allactives = sorted(allactives, reverse = True)
777 if self.activenb == 0 :
779 if len(allactives) <= nbmax :
780 log.info('nb = %i - eff min = %i ' % (len(allactives), allactives[-1][0]))
781 return [val[1] for val in allactives], allactives[-1][0]
783 effs = [val[0] for val in allactives]
784 if effs.count(effs[nbmax - 1]) > 1 :
785 lim = effs[nbmax - 1] + 1
789 stop = effs.index(lim)
796 log.info('nb actives = %i - eff min = %i ' % (stop + 1, lim))
797 return [val[1] for val in allactives[0:stop]], lim
799 def make_and_write_profile(self, actives, ucecl, fileout, uci = False) :
800 log.info('formes/classes')
802 tab = [[lem] + [len(set(self.getlemucis(lem)).intersection(classe)) for classe in ucecl] for lem in actives]
804 tab = [[lem] + [len(set(self.getlemuces(lem)).intersection(classe)) for classe in ucecl] for lem in actives]
805 tab = [[line[0]] + [repr(val) for val in line[1:]] for line in tab if sum(line[1:]) >= 3]
806 with open(fileout, 'w', encoding='utf8') as f :
807 f.write('\n'.join([';'.join(line) for line in tab]))
809 def make_etoiles(self) :
811 for uci in self.ucis :
812 etoiles.update(uci.etoiles[1:])
815 def make_themes(self):
817 for uci in self.ucis :
818 themes.update(uci.paras)
821 def make_etoiles_dict(self) :
822 etoiles = [et for uci in self.ucis for et in uci.etoiles[1:]]
824 for etoile in etoiles :
825 et = etoile.split('_')
828 endet = '_'.join(et[1:])
829 if etoile in det[et[0]] :
830 det[et[0]][etoile] += 1
832 det[et[0]][etoile] = 1
837 endet = '_'.join(et[1:])
838 det[et[0]] = {etoile :1}
843 def make_theme_dict(self):
844 themes = [val for uci in self.ucis for val in uci.paras]
846 for theme in themes :
847 th = theme.split('_')
850 endth = '_'.join(th[1:])
851 if theme in det[th[0]] :
852 det[th[0]][theme] += 1
854 det[th[0]][theme] = 1
859 endth = '_'.join(th[1:])
860 det[th[0]] = {theme:1}
865 def make_etline(self, listet) :
866 etuces = [[] for et in listet]
867 for uci in self.ucis :
868 get = list(set(uci.etoiles).intersection(listet))
870 return '2 variables sur la meme ligne'
872 etuces[listet.index(get[0])] += [uce.ident for uce in uci.uces]
875 def make_and_write_profile_et(self, ucecl, fileout, uci = False) :
876 log.info('etoiles/classes')
878 etoileuces = self.getetoileuces()
880 etoileuces = self.getetoileucis()
881 etoileuces = dict([[et, etoileuces[et]] for et in etoileuces if len(etoileuces[et]) > 1])
882 with open(fileout, 'w', encoding='utf8') as f :
883 f.write('\n'.join([';'.join([et] + [repr(len(set(etoileuces[et]).intersection(classe))) for classe in ucecl]) for et in etoileuces])) #.encode(self.parametres['syscoding'])
884 #etoiles = self.make_etoiles()
885 #with open(fileout, 'w') as f :
886 # f.write('\n'.join([';'.join([etoile] + [`len(set(self.getucesfrometoile(etoile)).intersection(classe))` for classe in ucecl]) for etoile in etoiles]).encode(self.parametres['syscoding']))
888 def make_colored_corpus(self, uci = False) :
890 for i, lc in enumerate(self.lc) :
893 for uce in self.lc0 :
895 color = ['black'] + colors[len(self.lc) - 1]
897 <meta http-equiv="content-Type" content="text/html; charset=utf8" />
901 res = self.getalluces()
906 if self.iduces[uce[0]].uci != actuci :
907 actuci = self.iduces[uce[0]].uci
908 txt += '<br><hr>' + ' '.join(self.ucis[self.iduces[uce[0]].uci].etoiles) + '<br><br>'
909 txt += '<font color="%s">' % (color[ucecl[uce[0]]]) + uce[1] + '</font><br><br>'
911 txt += '<font color="%s">' % (color[ucecl[uce[0]]]) + uce[1] + '</font><br><br>'
913 res = self.getallucis()
916 if self.ucis[uce[0]].ident != actuci :
917 actuci = self.ucis[uce[0]].ident
918 txt += '<br><hr>' + ' '.join(self.ucis[self.ucis[uce[0]].ident].etoiles) + '<br><br>'
919 txt += '<font color="%s">' % (color[ucecl[uce[0]]]) + uce[1] + '</font><br><br>'
921 txt += '<font color="%s">' % (color[ucecl[uce[0]]]) + uce[1] + '</font><br><br>'
922 return txt + '\n</body></html>'
924 def make_cut_corpus(self, uci = False) :
927 res = self.getalluces()
932 if self.iduces[uce[0]].uci != actuci :
933 actuci = self.iduces[uce[0]].uci
934 txt += '\n' + ' '.join(self.ucis[self.iduces[uce[0]].uci].etoiles) + '\n'
935 txt += ''.join(['\n',uce[1],'\n'])
937 txt += ''.join(['\n',uce[1],'\n'])
939 res = self.getallucis()
942 if self.ucis[uce[0]].ident != actuci :
943 actuci = self.ucis[uce[0]].ident
944 txt += '\n' + ' '.join(self.ucis[self.ucis[uce[0]].ident].etoiles) + '\n'
945 txt += ''.join(['\n',uce[1],'\n'])
947 txt += ''.join(['\n',uce[1],'\n'])
950 def count_from_list(self, l, d) :
958 def count_from_list_cl(self, l, d, a, clnb) :
967 def find_segments(self, taille_segment, taille_limite) :
969 for uce in self.getalluces() :
971 d = self.count_from_list([' '.join(uce[i:i+taille_segment]) for i in range(len(uce)-(taille_segment - 1))], d)
972 l = [[d[val], val] for val in d if d[val] >= 3]
975 if len(l) > taille_limite :
976 l = l[-taille_limite:]
979 def find_segments_in_classe(self, list_uce, taille_segment, taille_limite, uci = False):
982 concorde = self.getconcorde
984 concorde = self.getuciconcorde
985 for uce in concorde(list_uce) :
987 d =self.count_from_list([' '.join(uce[i:i+taille_segment]) for i in range(len(uce)-(taille_segment - 1))], d)
988 l = [[d[val], val, taille_segment] for val in d if d[val] >= 3]
991 if len(l) > taille_limite :
992 l = l[-taille_limite:]
995 def make_segments_profile(self, fileout, lenmin = 3, lenmax = 10, effmin = 50, lem = False) :
997 for b, classe in enumerate(self.lc) :
998 for uce in self.getconcorde(classe) :
1001 uce = [self.formes[forme].lem for forme in uce]
1002 for taille_segment in range(lenmin,lenmax) :
1003 d =self.count_from_list_cl([' '.join(uce[i:i+taille_segment]) for i in range(len(uce)-(taille_segment - 1))], d, b, len(self.lc))
1004 result = [[seg] + [str(val) for val in d[seg]] for seg in d if sum(d[seg]) >= effmin]
1005 with open(fileout, 'w', encoding='utf8') as f :
1006 f.write('\n'.join([';'.join(line) for line in result]))
1008 def make_proftype(self, outf) :
1010 for lem in self.lems :
1011 gram = self.lems[lem].gram
1012 if not gram in res :
1013 res[gram] = [0 for val in self.lc]
1014 lemuceeff = self.getlemuceseff(lem)
1015 for i, classe in enumerate(self.lc) :
1016 concern = set(classe).intersection(list(lemuceeff.keys()))
1017 res[gram][i] += sum([lemuceeff[uce] for uce in concern])
1018 res = [[gram] + [repr(val) for val in res[gram]] for gram in res]
1020 with open(outf, 'w', encoding='utf8') as f :
1021 f.write('\n'.join([';'.join(line) for line in res]))
1023 def make_ucecl_from_R(self, filein) :
1024 with open(filein, 'r') as f :
1029 line = line.replace('\n', '').replace('"', '').split(';')
1030 self.lc.append([int(line[0]) - 1, int(line[1])])
1031 classesl = [val[1] for val in self.lc]
1032 clnb = max(classesl)
1033 self.lc = sorted(self.lc, key=itemgetter(1))
1034 self.lc = [[uce[0] for uce in self.lc if uce[1] == i] for i in range(clnb+1)]
1035 self.lc0 = self.lc.pop(0)
1038 def get_stat_by_cluster(self, outf, lclasses = None) :
1039 log.info('get_stat_by_cluster')
1040 if lclasses is None :
1043 occurrences = dict([[i + 1, 0] for i in range(len(lclasses))])
1044 formescl = dict([[i + 1, 0] for i in range(len(lclasses))])
1045 hapaxcl = dict([[i + 1, 0] for i in range(len(lclasses))])
1046 lenclasses = dict([[i+1,len(cl)] for i, cl in enumerate(lclasses)])
1047 sets = [set(cl) for cl in lclasses]
1048 for forme in self.formes :
1049 formeuceeff = self.getformeuceseff(forme)
1050 for i, classe in enumerate(lclasses) :
1051 concern = sets[i].intersection(list(formeuceeff.keys()))
1053 occurrences[i+1] += sum([formeuceeff[uce] for uce in concern])
1055 if self.formes[forme].freq == 1 :
1057 log.info('%f' % (time() - t1))
1058 if outf is not None :
1059 toprint = '\n'.join([';'.join([repr(i), repr(occurrences[i]), repr(formescl[i]), repr(hapaxcl[i]), repr(lenclasses[i]), repr(float(hapaxcl[i])/float(formescl[i]))]) for i in occurrences])
1060 with open(outf, 'w', encoding='utf8') as f :
1063 return [[repr(occurrences[i]), repr(formescl[i]), repr(hapaxcl[i]), repr(lenclasses[i]), repr(float(hapaxcl[i])/float(formescl[i]))] for i in occurrences]
1065 def get_stat_by_et(self, outf, etoiles) :
1066 lclasses = [self.getucesfrometoile(etoile) for etoile in etoiles]
1067 stats = self.get_stat_by_cluster(None, lclasses)
1068 stats = [[etoiles[i]] + val for i, val in enumerate(stats)]
1070 def gethapaxbyet(self, etoiles) :
1071 hapaxuces = [self.getlemuces(forme)[0] for forme in self.lems if self.lems[forme].freq == 1]
1073 for uce in hapaxuces :
1074 if uce in hucesdict :
1078 etuces = [[] for et in etoiles]
1079 for uci in self.ucis :
1080 get = list(set(uci.etoiles).intersection(etoiles))
1082 return '2 variables sur la meme ligne'
1084 etuces[etoiles.index(get[0])] += [uce.ident for uce in uci.uces]
1085 etuces = [set(val) for val in etuces]
1086 return [sum([hucesdict[uce] for uce in list(etuce.intersection(hapaxuces))]) for etuce in etuces]
1088 def gethapaxuces(self) :
1089 hapaxuces = [self.getlemuces(forme)[0] for forme in self.lems if self.lems[forme].freq == 1]
1090 hapax = [forme for forme in self.lems if self.lems[forme].freq == 1]
1092 for i,uce in enumerate(hapaxuces) :
1093 if uce in hucesdict :
1094 hucesdict[uce][0] += 1
1095 hucesdict[uce][1].append(hapax[i])
1097 hucesdict[uce] = [1,[hapax[i]]]
1099 for uce in hucesdict :
1100 if hucesdict[uce][0] in huces :
1101 huces[hucesdict[uce][0]].append(uce)
1103 huces[hucesdict[uce][0]] = [uce]
1104 huces = list(zip(huces, list(huces.values())))
1105 huces.sort(reverse=True)
1109 for nb in huces[0:4] :
1110 txt += "<p><h2>%i hapax par uce</h2><p>\n" % nb[0]
1112 res = self.getconcorde([uce])
1114 ucetxt = ' ' + row[1] + ' '
1116 for hap in hucesdict[uce][1] :
1117 laforme = self.getforme([forme for forme in self.lems[hap].formes][0]).forme
1118 ucetxt = ucetxt.replace(' '+laforme+' ', ' <font color=red>'+laforme+'</font> ')
1119 txt += '<p><b>' + ' '.join(self.getetbyuceid(uceid)) + '</b></p>'
1120 txt += '<p>'+ucetxt+'</p>\n'
1124 with open('/tmp/testhapxuce.html','w', encoding='utf8') as f :
1127 def export_dictionary(self, fileout, syscoding) :
1128 listformes = [[self.formes[forme].freq, forme, self.formes[forme].lem, self.formes[forme].gram] for forme in self.formes]
1129 listformes.sort(reverse = True)
1130 listformes = [forme[1:] + [repr(forme[0])] for forme in listformes]
1131 with open(fileout, 'w', encoding='utf8') as f :
1132 f.write('\n'.join(['\t'.join(forme) for forme in listformes]))
1134 def export_lems(self, fileout, syscoding) :
1135 self.make_idformes()
1136 listlem = [[lem, '\t'.join(['\t'.join([self.idformes[forme].forme, repr(self.lems[lem].formes[forme])]) for forme in self.lems[lem].formes])] for lem in self.lems]
1138 with open(fileout, 'w', encoding='utf8') as f :
1139 f.write('\n'.join(['\t'.join(lem) for lem in listlem]))
1143 def __init__(self, corpus) :
1144 ucinb = corpus.getucinb()
1145 ucisize = corpus.getucisize()
1146 ucimean = float(sum(ucisize))/float(ucinb)
1147 detoile = corpus.make_etoiles_dict()
1150 def __init__(self, iduci, line, paraset = None) :
1152 self.etoiles = line.split()
1154 if paraset is not None :
1155 self.paras = paraset.split()
1160 def __init__(self, iduce, idpara, iduci) :
1166 def __init__(self, word, gramtype, idword, lem = None, freq = None) :
1169 self.gram = gramtype
1172 if freq is not None :
1178 def __init__(self, parent, forme) :
1179 self.formes = {forme.ident : forme.freq}
1180 self.gram = forme.gram
1181 self.freq = forme.freq
1182 self.act = forme.act
1184 def add_forme(self, forme) :
1185 self.formes[forme.ident] = forme.freq
1186 self.freq += forme.freq
1189 def decouperlist(chaine, longueur, longueurOptimale) :
1191 on part du dernier caractère, et on recule jusqu'au début de la chaîne.
1192 Si on trouve un '$', c'est fini.
1193 Sinon, on cherche le meilleur candidat. C'est-à-dire le rapport poids/distance le plus important.
1195 separateurs = [['.', 6.0], ['?', 6.0], ['!', 6.0], ['£$£', 6.0], [':', 5.0], [';', 4.0], [',', 1.0], [' ', 0.01]]
1196 dsep = dict([[val[0],val[1]] for val in separateurs])
1197 trouve = False # si on a trouvé un bon séparateur
1198 iDecoupe = 0 # indice du caractere ou il faut decouper
1199 longueur = min(longueur, len(chaine) - 1)
1200 chaineTravail = chaine[:longueur + 1]
1202 meilleur = ['', 0, 0] # type, poids et position du meilleur separateur
1204 indice = chaineTravail.index('$')
1206 iDecoupe = indice - 1
1211 caractere = chaineTravail[nbCar]
1212 distance = abs(longueurOptimale - nbCar) + 1
1213 meilleureDistance = abs(longueurOptimale - meilleur[2]) + 1
1214 if caractere in dsep :
1215 if (float(dsep[caractere]) / distance) > (float(meilleur[1]) / meilleureDistance) :
1216 meilleur[0] = caractere
1217 meilleur[1] = dsep[caractere]
1222 if (float(dsep[' ']) / distance) > (float(meilleur[1]) / meilleureDistance) :
1224 meilleur[1] = dsep[' ']
1232 #if meilleur[0] != ' ' :
1233 # fin = chaine[iDecoupe + 1:]
1234 # retour = chaineTravail[:iDecoupe]
1237 fin = chaine[iDecoupe + 1:]
1238 retour = chaineTravail[:iDecoupe + 1]
1239 return len(retour) > 0, retour, fin
1240 # si on a rien trouvé
1241 return False, chaine, ''
1243 def testetoile(line) :
1244 return line.startswith('****')
1247 return line[0:4].isdigit() and '*' in line
1249 def prep_txtlist(txt) :
1250 return txt.split() + ['$']
1252 def prep_txtcharact(txt) :
1258 Class for building a corpus
1261 def __init__(self, infile, parametres_corpus, lexique = None, expressions = None, dlg = None) :
1262 log.info('begin building corpus...')
1263 self.lexique = lexique
1264 self.expressions = expressions
1266 self.corpus = Corpus(self, parametres_corpus)
1267 self.infile = infile
1269 self.lim = parametres_corpus.get('lim', 1000000)
1270 self.encoding = parametres_corpus['encoding']
1271 self.corpus.pathout = PathOut(filename = parametres_corpus['originalpath'], dirout = parametres_corpus['pathout'])
1272 self.corpus.pathout.createdir(parametres_corpus['pathout'])
1273 self.corpus.parametres['uuid'] = str(uuid4())
1274 self.corpus.parametres['corpus_name'] = parametres_corpus['corpus_name'] #os.path.split(self.corpus.parametres['pathout'])[1]
1275 self.corpus.parametres['type'] = 'corpus'
1276 if self.corpus.parametres['keep_ponct'] :
1277 self.ponctuation_espace = [' ', '']
1279 self.ponctuation_espace = [' ','.', '£$£', ';', '?', '!', ',', ':','']
1281 self.tolist = self.corpus.parametres.get('tolist', 0)
1288 def prep_makeuce(self) :
1289 method = self.corpus.parametres.get('ucemethod', 0)
1291 self.decouper = decouperlist
1292 self.prep_txt = prep_txtlist
1293 self.ucesize = self.corpus.parametres.get('ucesize', 40)
1295 self.decouper = decoupercharact
1296 self.prep_txt = prep_txtcharact
1297 self.ucesize = self.corpus.parametres.get('ucesize', 240)
1298 log.info('method uce : %s' % method)
1303 self.read_corpus(self.infile)
1304 except Warning as args :
1305 log.info('pas kool %s' % args)
1309 self.corpus.parametres['ira'] = self.corpus.pathout['Corpus.cira']
1310 self.time = time() - t1
1312 DoConf().makeoptions(['corpus'],[self.corpus.parametres], self.corpus.pathout['Corpus.cira'])
1313 log.info('time : %f' % (time() - t1))
1316 self.conn_f = sqlite3.connect(self.corpus.pathout['formes.db'])
1317 self.cf = self.conn_f.cursor()
1318 self.cf.execute('CREATE TABLE IF NOT EXISTS uces (id INTEGER, uces TEXT);')
1319 self.cf.execute('CREATE TABLE IF NOT EXISTS eff (id INTEGER, eff TEXT);')
1320 self.conn_f.commit()
1321 self.cf = self.conn_f.cursor()
1322 self.cf.execute('PRAGMA temp_store=MEMORY;')
1323 self.cf.execute('PRAGMA journal_mode=MEMORY;')
1324 self.cf.execute('PRAGMA synchronous = OFF;')
1325 self.cf.execute('begin')
1326 self.conn = sqlite3.connect(self.corpus.pathout['uces.db'])
1327 self.c = self.conn.cursor()
1328 self.c.execute('CREATE TABLE IF NOT EXISTS uces (id INTEGER, uces TEXT);')
1330 self.c = self.conn.cursor()
1331 self.c.execute('PRAGMA temp_store=MEMORY;')
1332 self.c.execute('PRAGMA journal_mode=MEMORY;')
1333 self.c.execute('PRAGMA synchronous = OFF;')
1334 self.c.execute('begin')
1337 #commit index and close db
1339 self.conn_f.commit()
1340 self.cf.execute('CREATE INDEX iduces ON uces (id);')
1341 self.cf.execute('CREATE INDEX ideff ON eff (id);')
1345 self.conn_corpus = sqlite3.connect(self.corpus.pathout['corpus.db'])
1346 self.ccorpus = self.conn_corpus.cursor()
1347 self.ccorpus.execute('CREATE TABLE IF NOT EXISTS etoiles (uci INTEGER, et TEXT, paras TEXT);')
1348 self.ccorpus.execute('CREATE TABLE IF NOT EXISTS luces (uci INTEGER, para INTEGER, uce INTEGER);')
1349 self.ccorpus.execute('CREATE TABLE IF NOT EXISTS formes (ident INTEGER, forme TEXT, lem TEXT, gram TEXT, freq INTEGER);')
1350 self.conn_corpus.commit()
1351 self.ccorpus = self.conn_corpus.cursor()
1352 self.ccorpus.execute('PRAGMA temp_store=MEMORY;')
1353 self.ccorpus.execute('PRAGMA journal_mode=MEMORY;')
1354 self.ccorpus.execute('PRAGMA synchronous = OFF;')
1355 self.ccorpus.execute('begin')
1356 self.backup_corpus()
1357 self.ccorpus.execute('CREATE INDEX iduci ON luces (uci);')
1358 self.conn_corpus.commit()
1359 self.conn_corpus.close()
1360 #self.corpus.parametres['corpus_ira'] = self.corpus.pathout['corpus.cira']
1362 def buildcleans(self) :
1363 if self.corpus.parametres.get('lower', 1) :
1364 self.cleans.append(self.dolower)
1365 if self.corpus.parametres.get('firstclean', 1) :
1366 self.cleans.append(self.firstclean)
1367 if self.corpus.parametres['charact'] :
1368 self.rule = self.corpus.parametres.get('keep_caract', "^a-zA-Z0-9àÀâÂäÄáÁéÉèÈêÊëËìÌîÎïÏòÒôÔöÖùÙûÛüÜçÇßœŒ’ñ.:,;!?*'_")
1369 self.cleans.append(self.docharact)
1370 if self.corpus.parametres.get('expressions', 1) :
1371 self.cleans.append(self.make_expression)
1372 if self.corpus.parametres.get('apos', 1) :
1373 self.cleans.append(self.doapos)
1374 if self.corpus.parametres.get('tiret', 1):
1375 self.cleans.append(self.dotiret)
1377 def make_expression(self,txt) :
1378 exp = list(self.expressions.keys())
1379 exp.sort(reverse=True)
1380 for expression in exp :
1381 if expression in txt :
1382 txt = txt.replace(expression, self.expressions[expression][0])
1385 def dolower(self, txt) :
1388 def docharact(self, txt) :
1389 #rule = u"^a-zA-Z0-9àÀâÂäÄáÁéÉèÈêÊëËìÌîÎïÏòÒôÔöÖùÙûÛüÜçÇßœŒ’ñ.:,;!?*'_-"
1390 list_keep = "[" + self.rule + "]+"
1391 return re.sub(list_keep, ' ', txt)
1393 def doapos(self, txt) :
1394 return txt.replace('\'', ' ')
1396 def dotiret(self, txt) :
1397 return txt.replace('-', ' ')
1399 def firstclean(self, txt) :
1400 txt = txt.replace('’',"'")
1401 txt = txt.replace('œ', 'oe')
1402 return txt.replace('...',' £$£ ').replace('?',' ? ').replace('.',' . ').replace('!', ' ! ').replace(',',' , ').replace(';', ' ; ').replace(':',' : ').replace('…', ' £$£ ')
1404 def make_cleans(self, txt) :
1405 for clean in self.cleans :
1409 def backup_uce(self) :
1410 if self.corpus.idformesuces != {} :
1411 log.info('backup %i' % len(self.corpus.idformesuces))
1412 touce = [(repr(forme), ' '.join([repr(val) for val in list(self.corpus.idformesuces[forme].keys())])) for forme in self.corpus.idformesuces]
1413 toeff = [(repr(forme), ' '.join([repr(val) for val in list(self.corpus.idformesuces[forme].values())])) for forme in self.corpus.idformesuces]
1414 self.cf.executemany('INSERT INTO uces VALUES (?,?);', touce)
1415 self.cf.executemany('INSERT INTO eff VALUES (?,?);', toeff)
1416 self.corpus.idformesuces = {}
1419 def backup_corpus(self) :
1420 log.info('start backup corpus')
1422 for uci in self.corpus.ucis :
1423 self.ccorpus.execute('INSERT INTO etoiles VALUES (?,?,?);' ,(uci.ident,' '.join(uci.etoiles), ' '.join(uci.paras,)))
1424 for uce in uci.uces :
1425 self.ccorpus.execute('INSERT INTO luces VALUES (?,?,?);',(repr(uci.ident),repr(uce.para),repr(uce.ident),))
1426 for forme in self.corpus.formes :
1427 self.ccorpus.execute('INSERT INTO formes VALUES (?,?,?,?,?);', (repr(self.corpus.formes[forme].ident), forme, self.corpus.formes[forme].lem, self.corpus.formes[forme].gram, repr(self.corpus.formes[forme].freq),))
1428 log.info('%f' % (time() - t))
1430 def dofinish(self) :
1431 self.corpus.parametres['date'] = datetime.datetime.now().ctime()
1432 minutes, seconds = divmod(self.time, 60)
1433 hours, minutes = divmod(minutes, 60)
1434 self.corpus.parametres['time'] = '%.0fh %.0fm %.0fs' % (hours, minutes, seconds)
1435 self.corpus.parametres['ucinb'] = self.corpus.getucinb()
1436 self.corpus.parametres['ucenb'] = self.corpus.getucenb()
1437 self.corpus.parametres['occurrences'] = self.corpus.gettotocc()
1438 self.corpus.parametres['formesnb'] = len(self.corpus.formes)
1439 hapaxnb = self.corpus.gethapaxnb()
1440 pourhapaxf = (float(hapaxnb) / len(self.corpus.formes)) * 100
1441 pourhapaxocc = (float(hapaxnb) / self.corpus.parametres['occurrences']) * 100
1442 self.corpus.parametres['hapax'] = '%i - %.2f %% des formes - %.2f %% des occurrences' % (hapaxnb, pourhapaxf, pourhapaxocc)
1444 class BuildSubCorpus(BuildCorpus):
1446 def __init__(self, corpus, parametres, dlg = None) :
1447 log.info('begin subcorpus...')
1451 self.corpus = Corpus(self, {'type' : 'corpus', 'originalpath' : corpus.parametres['originalpath'], 'encoding' : corpus.parametres['encoding']})
1453 self.parametres = parametres
1454 self.encoding = corpus.parametres['encoding']
1455 self.corpus.parametres['corpus_name'] = parametres['corpus_name']
1456 self.corpus.pathout = PathOut(filename = corpus.parametres['originalpath'], dirout = parametres['pathout'])
1457 self.corpus.pathout.createdir(parametres['pathout'])
1458 self.corpus.parametres['pathout'] = parametres['pathout']
1459 self.corpus.parametres['meta'] = parametres.get('meta', False)
1460 self.corpus.parametres['uuid'] = str(uuid4())
1461 if parametres.get('frommeta', False) :
1462 print('make subtexts')
1463 self.corpus.ucis = [CopyUci(uci) for uci in self.ori.ucis if set(parametres['meta']).intersection(uci.etoiles) != set()]
1464 elif parametres.get('fromtheme', False) :
1465 print('make subtexts from theme')
1467 for uci in self.ori.ucis :
1468 if uci.paras != [] :
1471 for et in uci.paras :
1472 if et in parametres['meta'] :
1473 newuce += [CopyUce(uce) for uce in uci.uces if uce.para == idpara]
1479 nuci.paras = newpara
1480 self.corpus.ucis.append(nuci)
1483 elif parametres.get('fromclusters', False) :
1484 self.parametres['uceids'] = [st for i in self.parametres['meta'] for st in self.parametres['lc'][i]]
1486 elif parametres.get('fromuceids', False) :
1492 def fromuceids(self):
1494 dictucekeep = dict(list(zip(self.parametres['uceids'], self.parametres['uceids'])))
1496 for uci in self.ori.ucis :
1497 if uci.paras == [] :
1498 keepuces = [CopyUce(uce) for uce in uci.uces if uce.ident in dictucekeep]
1501 nuci.uces = keepuces
1502 self.corpus.ucis.append(nuci)
1507 for et in uci.paras :
1508 keepuces = [CopyUce(uce) for uce in uci.uces if uce.ident in dictucekeep]
1516 nuci.paras = newpara
1517 self.corpus.ucis.append(nuci)
1519 def read_corpus(self, infile = None):
1520 self.olduceid = [uce.ident for uci in self.corpus.ucis for uce in uci.uces]
1526 print('redo text, para and st ident')
1527 for uci in self.corpus.ucis :
1528 uci.ident = ident_uci
1530 for uce in uci.uces :
1532 if uce.para != lastpara :
1535 uce.para = ident_para
1537 uce.para = ident_para
1538 newuceident[uce.ident] = ident_uce
1539 uce.ident = ident_uce
1541 print('backup st text and forms')
1542 for row in self.ori.getconcorde(self.olduceid) :
1543 self.c.execute('INSERT INTO uces VALUES(?,?);', (repr(newuceident[row[0]]), row[1]))
1544 for word in row[1].split() :
1545 self.corpus.add_word_from_forme(self.ori.formes[word], newuceident[row[0]])
1549 class BuildFromAlceste(BuildCorpus) :
1551 def read_corpus(self, infile) :
1553 if self.dlg is not None :
1554 self.dlg.Pulse('textes : 0 - segments : 0')
1557 if self.corpus.parametres['ucimark'] == 0 :
1558 self.testuci = testetoile
1559 elif self.corpus.parametres['ucimark'] == 1 :
1560 self.testuci = testint
1566 with codecs.open(infile, 'r', self.encoding) as f :
1567 for linenb, line in enumerate(f) :
1568 line = line.rstrip('\n\r')#FIXME .lstrip(codecs.BOM).lstrip(codecs.BOM_UTF8)
1569 if self.testuci(line) :
1572 #doc = nlp(' '.join(txt))
1573 #print([[word, word.pos_, word.lemma_] for word in doc])
1574 iduce, idpara = self.treattxt(txt, iduce, idpara, iduci - 1)
1576 self.corpus.ucis.append(Uci(iduci, line))
1579 if self.corpus.ucis[-1].uces == [] :
1580 log.info('Empty text : %i' % linenb)
1582 self.corpus.ucis.pop()
1583 self.corpus.ucis.append(Uci(iduci, line))
1584 if self.dlg is not None :
1585 if not (iduci + 1) % 10 :
1586 self.dlg.Pulse('textes : %i - segments : %i' % (iduci + 1, iduce +1))
1587 elif line.startswith('-*') :
1590 iduce, idpara = self.treattxt(txt, iduce, idpara, iduci)
1593 self.corpus.ucis[-1].paras.append(line.split()[0])
1595 raise Exception('paragrapheOT %i' % linenb)
1596 elif line.strip() != '' and iduci != -1 :
1598 if txt != [] and iduci != -1 :
1599 iduce, idpara = self.treattxt(txt, iduce, idpara, iduci)
1604 self.corpus.ucis.pop()
1605 log.info(Exception("Empty text %i" % linenb))
1607 raise Exception('EmptyText %i' % linenb)
1608 if iduci != -1 and iduce != -1:
1611 log.info(_("No Text in corpus. Are you sure of the formatting ?"))
1612 raise Exception('TextBeforeTextMark %i' % linenb)
1613 except UnicodeDecodeError :
1614 raise Exception("CorpusEncoding")
1616 def treattxt(self, txt, iduce, idpara, iduci) :
1617 if self.corpus.parametres.get('ucemethod', 0) == 2 and self.corpus.parametres['douce']:
1618 txt = 'laphrasepoursplitter'.join(txt)
1619 txt = self.make_cleans(txt)
1620 txt = ' '.join([val for val in txt.split() if val not in self.ponctuation_espace])
1621 ucetxt = txt.split('laphrasepoursplitter')
1624 txt = self.make_cleans(txt)
1625 ucetxt = self.make_uces(txt, self.corpus.parametres['douce'])
1626 if self.corpus.ucis[-1].paras == [] :
1630 self.corpus.ucis[-1].uces.append(Uce(iduce, idpara, iduci))
1631 self.c.execute('INSERT INTO uces VALUES(?,?);', (repr(iduce),uce))
1632 if not self.tolist :
1638 self.corpus.add_word(word)
1639 log.debug(' '.join([repr(iduci),repr(idpara),repr(iduce)]))
1640 if self.last > self.lim :
1643 return iduce, idpara
1645 def make_uces(self, txt, douce = True, keep_ponct = False) :
1646 txt = ' '.join(txt.split())
1649 reste, texte_uce, suite = self.decouper(self.prep_txt(txt), self.ucesize + 15, self.ucesize)
1651 uce = ' '.join([val for val in texte_uce if val not in self.ponctuation_espace])
1654 reste, texte_uce, suite = self.decouper(suite, self.ucesize + 15, self.ucesize)
1655 uce = ' '.join([val for val in texte_uce if val not in self.ponctuation_espace])
1660 return [' '.join([val for val in txt.split() if val not in self.ponctuation_espace])]
1662 #decouper (list_sep)
1663 #make_uces (decouper)
1664 #treat_txt (make_uces)
1669 def __init__(self, parent, dlg = None) :
1670 self.parent = parent
1672 parametres = DoConf(os.path.join(self.parent.UserConfigPath,'corpus.cfg')).getoptions('corpus')
1673 parametres['pathout'] = PathOut(parent.filename, 'corpus').mkdirout()
1674 parametres['corpus_name'] = os.path.split(parametres['pathout'])[1]
1675 dial = CorpusPref(parent, parametres)
1676 dial.CenterOnParent()
1677 dial.txtpath.SetLabel(parent.filename)
1678 #dial.repout_choices.SetValue(parametres['pathout'])
1679 self.res = dial.ShowModal()
1680 if self.dlg is not None :
1681 self.dlg = progressbar(self.parent, self.dlg)
1682 if self.res == 5100 :
1683 parametres = dial.doparametres()
1684 parametres['originalpath'] = parent.filename
1685 PathOut().createdir(parametres['pathout'])
1686 if parametres.get('dictionary', False) :
1687 filein = parametres['dictionary']
1690 if dial.corpusname.GetValue() != '' :
1691 parametres['corpus_name'] = dial.corpusname.GetValue()
1693 ReadLexique(self.parent, lang = parametres['lang'], filein = filein)
1694 if parametres['lang'] != 'other' and os.path.exists(self.parent.DictPath.get(parametres['lang']+'_exp', 'french_exp')):
1695 self.parent.expressions = ReadDicoAsDico(self.parent.DictPath.get(parametres['lang']+'_exp', 'french_exp'))
1697 self.parent.expressions = {}
1698 self.parametres = parametres
1701 if self.dlg is not None :
1704 def doanalyse(self) :
1705 return BuildFromAlceste(self.parent.filename, self.parametres, self.parent.lexique, self.parent.expressions, dlg = self.dlg).corpus
1709 def __init__(self, parent, corpus, parametres = None, dlg = None):
1710 self.parent = parent
1713 corpus_name = 'Sub' + corpus.parametres['corpus_name']
1714 if dlg is not None :
1715 busy = wx.BusyInfo(_("Please wait..."), self)
1717 parametres['corpus_name'] = corpus_name
1718 if parametres.get('frommeta', False) :
1719 parametres['meta'] = corpus.make_etoiles()
1720 elif parametres.get('fromtheme', False) :
1721 parametres['meta'] = corpus.make_themes()
1722 elif parametres.get('fromclusters', False) :
1723 parametres['meta'] = [' '.join(['classe', repr(i)]) for i in range(1,parametres['clnb'] + 1)]
1725 parametres['meta'] = []
1726 if 'fromclusters' not in parametres :
1727 parametres['meta'].sort()
1728 if dlg is not None :
1730 dial = SubTextFromMetaDial(parent, parametres)
1731 self.res = dial.ShowModal()
1732 if self.res == 5100 :
1733 if dial.subcorpusname.GetValue() != '' :
1734 corpus_name = ''.join([l for l in dial.subcorpusname.GetValue() if l.isalnum() or l in ['_']])
1735 if corpus_name != '' :
1736 parametres['corpus_name'] = corpus_name
1738 parametres['corpus_name'] = 'Sub' + corpus.parametres['corpus_name']
1739 pathout = os.path.join(corpus.parametres['pathout'], parametres['corpus_name'])
1741 while os.path.exists(pathout + '_%i' % i) :
1743 parametres['pathout'] = pathout + '_%i' % i
1744 meta = dial.m_listBox1.GetSelections()
1745 if not 'fromclusters' in parametres :
1746 parametres['meta'] = [parametres['meta'][val] for val in meta]
1748 parametres['meta'] = meta
1749 self.parametres = parametres
1754 def doanalyse(self):
1755 return BuildSubCorpus(self.ori, parametres = self.parametres, dlg = self.dlg).corpus
1757 class BuildMergeFromClusters(BuildCorpus):
1759 def __init__(self, analyses, parametres, dlg = None) :
1760 log.info('begin subcorpus...')
1763 self.corpus = Corpus(self, {'type' : 'corpus', 'originalpath' : 'MergeFromClusters', 'encoding' : 'merge'})
1765 self.analyses = analyses
1767 self.parametres = parametres
1768 #self.encoding = corpus.parametres['encoding']
1769 self.corpus.parametres['corpus_name'] = parametres['corpus_name']
1770 self.corpus.pathout = PathOut(filename = 'MFC', dirout = parametres['pathout'])
1771 self.corpus.pathout.createdir(parametres['pathout'])
1772 self.corpus.parametres['pathout'] = parametres['pathout']
1773 self.corpus.parametres['meta'] = parametres.get('meta', False)
1774 self.corpus.parametres['uuid'] = str(uuid4())
1775 for i, analyse in enumerate(analyses) :
1778 corpus_uuid = analyse['corpus']
1779 #if corpus_uuid not in self.parent.history.openedcorpus :
1780 irapath = parametres['corpusira'][i]
1781 corpus = Corpus(self, parametres = DoConf(irapath).getoptions('corpus'), read = True)
1782 ucepath = os.path.join(analyse['pathout'], 'uce.csv')
1783 corpus.make_ucecl_from_R(ucepath)
1785 for j, cl in enumerate(parametres['clusters'][i]) :
1786 #print cl, self.ori.lc[cl-1]
1787 self.parametres['uceids'] = self.ori.lc[cl-1]#[st for st in self.ori['lc'][cl-1]]
1788 self.lcl[i] += self.ori.lc[cl-1]
1789 self.et = parametres['newet'][i][j]
1795 def fromuceids(self):
1797 dictucekeep = dict(list(zip(self.parametres['uceids'], self.parametres['uceids'])))
1799 for uci in self.ori.ucis :
1800 if uci.paras == [] :
1801 keepuces = [CopyUce(uce) for uce in uci.uces if uce.ident in dictucekeep]
1804 nuci.uces = keepuces
1805 nuci.etoiles.append(self.et)
1806 nuci.analyseid = self.analyseid
1807 self.corpus.ucis.append(nuci)
1812 for et in uci.paras :
1813 keepuces = [CopyUce(uce) for uce in uci.uces if uce.ident in dictucekeep]
1821 nuci.paras = newpara
1822 nuci.etoiles.append(self.et)
1823 nuci.analyseid = self.analyseid
1824 self.corpus.ucis.append(nuci)
1825 #print nuci.etoiles, nuci.ident, nuci.uces
1827 def read_corpus(self, infile = None):
1828 #self.olduceid = [uce.ident for uci in self.corpus.ucis for uce in uci.uces]
1834 print('redo text, para and st ident')
1835 for uci in self.corpus.ucis :
1836 #print uci.ident, ident_uci, [uce.ident for uce in uci.uces], uci.etoiles
1837 uci.ident = ident_uci
1839 for uce in uci.uces :
1841 if uce.para != lastpara :
1844 uce.para = ident_para
1846 uce.para = ident_para
1847 newuceident['%i-%i' %(uci.analyseid, uce.ident)] = ident_uce
1848 uce.ident = ident_uce
1851 print('backup st text and forms')
1853 for i, analyse in enumerate(self.analyses) :
1854 #print analyse, self.parametres['corpusira']
1855 irapath = self.parametres['corpusira'][i]
1856 old = Corpus(self, parametres = DoConf(irapath).getoptions('corpus'), read = True)
1857 for row in old.getconcorde(self.lcl[i]) :
1858 self.c.execute('INSERT INTO uces VALUES(?,?);', (newuceident['%i-%i' % (i,row[0])], row[1]))
1859 for word in row[1].split() :
1860 self.corpus.add_word_from_forme(old.formes[word], newuceident['%i-%i' % (i,row[0])])
1866 class MergeClusters :
1868 def __init__(self, parent, parametres = None, dlg = None):
1869 self.parent = parent
1872 corpus_name = 'MergeFromClusters'
1873 if dlg is not None :
1874 busy = wx.BusyInfo(_("Please wait..."), self)
1876 parametres['corpus_name'] = corpus_name
1877 if dlg is not None :
1879 dial = MergeClusterFrame(parent)
1880 dial.m_textCtrl4.SetValue(corpus_name)
1881 self.res = dial.ShowModal()
1882 if self.res == 5100 :
1887 if dial.m_textCtrl4.GetValue() != '' :
1888 corpus_name = ''.join([l for l in dial.m_textCtrl4.GetValue() if l.isalnum() or l in ['_']])
1889 if corpus_name != '' :
1890 parametres['corpus_name'] = corpus_name
1892 parametres['corpus_name'] = 'MergeFromClusters'
1893 for cl in dial.selected :
1895 #if corpus_uuid not in self.parent.history.openedcorpus :
1896 irapath = self.parent.history.corpus[corpus_uuid]['ira']
1897 #corpus = Corpus(self.parent, parametres = DoConf(irapath).getoptions('corpus'), read = True)
1898 #self.parent.history.openedcorpus[corpus_uuid] = corpus
1899 if cl[0] not in self.analyses :
1900 analyse = DoConf(dial.irapath[cl[0]]).getoptions()
1901 #ucepath = os.path.join(os.path.dirname(dial.irapath[cl[0]]), 'uce.csv')
1902 #corpus = copycorpus(self.parent.history.openedcorpus[corpus_uuid])
1903 #corpus.make_ucecl_from_R(ucepath)
1904 self.analyses[cl[0]] = analyse
1905 self.clusters[cl[0]] = [cl[2]]
1906 self.newet[cl[0]] = [dial.selected[cl]]
1907 self.corpusira[cl[0]] = irapath
1909 self.clusters[cl[0]].append(cl[2])
1910 self.newet[cl[0]].append(dial.selected[cl])
1911 analyses = [val for val in self.clusters]
1912 clusters = [self.clusters[val] for val in analyses]
1913 self.newet = [self.newet[val] for val in analyses]
1914 corpusira = [self.corpusira[val] for val in analyses]
1915 analyses = [self.analyses[val] for val in analyses]
1916 pathout = os.path.dirname(os.path.dirname(analyses[0]['pathout']))
1917 self.analyses = analyses
1918 pathout = os.path.join(pathout, parametres['corpus_name'])
1920 while os.path.exists(pathout + '_%i' % i) :
1922 parametres['pathout'] = pathout + '_%i' % i
1923 self.parametres = parametres
1924 self.parametres['clusters'] = clusters
1925 self.parametres['newet'] = self.newet
1926 self.parametres['corpusira'] = corpusira
1931 def doanalyse(self):
1932 return BuildMergeFromClusters(self.analyses, parametres = self.parametres, dlg = self.dlg).corpus