X-Git-Url: http://www.iramuteq.org/git?a=blobdiff_plain;f=corpus.py;h=489d4f18de1c71b1ce80f3699985b5cfefd62b21;hb=refs%2Fheads%2F3.0;hp=8518b251331a19c14f8f9c2be8d0e66d81674581;hpb=e950507e6d72b808f1f06131eaa561676eb27fad;p=iramuteq
diff --git a/corpus.py b/corpus.py
index 8518b25..555a034 100644
--- a/corpus.py
+++ b/corpus.py
@@ -1,25 +1,39 @@
# -*- coding: utf-8 -*-
#Author: Pierre Ratinaud
+#Copyright (c) 2008-2020 Pierre Ratinaud
+#modification pour python 3 : Laurent Mérat, 6x7 - mai 2020
+#License: GNU/GPL
+#------------------------------------
+# import des modules python
+#------------------------------------
import codecs
import os
-import gettext
-_ = gettext.gettext
import locale
import sys
from time import time
-from functions import decoupercharact, ReadDicoAsDico, DoConf, ReadLexique
import re
import sqlite3
import itertools
import logging
from operator import itemgetter
from uuid import uuid4
-from chemins import PathOut
-from dialog import CorpusPref, SubTextFromMetaDial
+import datetime
from copy import copy
+#------test spacy------------
+#import spacy
+#nlp = spacy.load("fr_core_news_lg")
+
+#------------------------------------
+# import des fichiers du projet
+#------------------------------------
+from functions import decoupercharact, ReadDicoAsDico, DoConf, ReadLexique, progressbar
+from chemins import PathOut
+from dialog import CorpusPref, SubTextFromMetaDial, MergeClusterFrame
from colors import colors
-import datetime
+
+import langue
+langue.run()
log = logging.getLogger('iramuteq.corpus')
@@ -36,24 +50,24 @@ def copycorpus(corpus) :
def CopyUce(uce) :
return Uce(uce.ident, uce.para, uce.uci)
-
def CopyUci(uci):
nuci = Uci(uci.ident, '')
nuci.etoiles = copy(uci.etoiles)
nuci.uces = [CopyUce(uce) for uce in uci.uces]
+ nuci.paras = copy(uci.paras)
return nuci
-
class Corpus :
+
"""Corpus class
list of text
"""
def __init__(self, parent, parametres = {}, read = False) :
self.parent = parent
self.parametres = parametres
- self.cformes = None
+ self.cformes = None
self.connformes = None
self.connuces = None
self.conncorpus = None
@@ -86,14 +100,14 @@ class Corpus :
gramtype = self.parent.lexique[word][1]
lem = self.parent.lexique[word][0]
elif word.isdigit() :
- gramtype = u'num'
+ gramtype = 'num'
lem = word
else :
- gramtype = u'nr'
+ gramtype = 'nr'
lem = word
self.formes[word] = Word(word, gramtype, len(self.formes), lem)
self.idformesuces[self.formes[word].ident] = {self.ucis[-1].uces[-1].ident : 1}
-
+
def add_word_from_forme(self, word, stident):
if word.forme in self.formes :
self.formes[word.forme].freq += 1
@@ -106,7 +120,7 @@ class Corpus :
self.idformesuces[self.formes[word.forme].ident] = {stident: 1}
else :
self.formes[word.forme] = Word(word.forme, word.gram, len(self.formes), word.lem)
- self.idformesuces[self.formes[word.forme].ident] = {stident : 1}
+ self.idformesuces[self.formes[word.forme].ident] = {stident : 1}
def conn_all(self):
"""connect corpus to db"""
@@ -130,33 +144,33 @@ class Corpus :
def read_corpus(self) :
log.info('read corpus')
- self.parametres['syscoding'] = sys.getdefaultencoding()
+ self.parametres['syscoding'] = 'utf8'
if self.conncorpus is None :
self.conn_all()
res = self.ccorpus.execute('SELECT * FROM etoiles;')
for row in res :
self.ucis.append(Uci(row[0], row[1], row[2]))
- uces = self.conncorpus.cursor().execute('SELECT * FROM luces where uci=?;',(`self.ucis[-1].ident`,))
+ uces = self.conncorpus.cursor().execute('SELECT * FROM luces where uci=?;',(repr(self.ucis[-1].ident),))
for uce in uces:
self.ucis[-1].uces.append(Uce(uce[2], uce[1], uce[0]))
res = self.ccorpus.execute('SELECT * FROM formes;')
self.formes = dict([[forme[1], Word(forme[1], forme[3], forme[0], lem = forme[2], freq = forme[4])] for forme in res])
self.ccorpus.close()
-
+
def getworduces(self, wordid) :
- if isinstance(wordid, basestring) :
+ if isinstance(wordid, str) :
wordid = self.formes[wordid].ident
- res = self.cformes.execute('SELECT uces FROM uces where id=? ORDER BY id;', (`wordid`,))
+ res = self.cformes.execute('SELECT uces FROM uces where id=? ORDER BY id;', (repr(wordid),))
return list(itertools.chain(*[[int(val) for val in row[0].split()] if not isinstance(row[0], int) else [row[0]] for row in res]))
-
+
def getworducis(self, wordid) :
res = self.getworduces(wordid)
return list(set([self.getucefromid(uce).uci for uce in res]))
def getformeuceseff(self, formeid) :
- if isinstance(formeid, basestring) :
+ if isinstance(formeid, str) :
formeid = self.formes[formeid].ident
- res = self.cformes.execute('SELECT uces FROM uces where id=? ORDER BY id;', (`formeid`,))
+ res = self.cformes.execute('SELECT uces FROM uces where id=? ORDER BY id;', (repr(formeid),))
uces = list(itertools.chain(*[[int(val) for val in row[0].split()] if not isinstance(row[0], int) else [row[0]] for row in res]))
query = 'SELECT eff FROM eff where id=%i ORDER BY id' % formeid
res = self.cformes.execute(query)
@@ -164,20 +178,48 @@ class Corpus :
formeuceeff = {}
for i, uce in enumerate(uces) :
formeuceeff[uce] = formeuceeff.get(uce, 0) + eff[i]
- return formeuceeff
+ return formeuceeff
def getlemuces(self, lem) :
- formesid = ', '.join([`val` for val in self.lems[lem].formes])
+ formesid = ', '.join([repr(val) for val in self.lems[lem].formes])
query = 'SELECT uces FROM uces where id IN (%s) ORDER BY id' % formesid
res = self.cformes.execute(query)
return list(set(list(itertools.chain(*[[int(val) for val in row[0].split()] if not isinstance(row[0], int) else [row[0]] for row in res]))))
+ def gettgenst(self, tgen):
+ formesid = []
+ for lem in tgen :
+ if lem in self.lems :
+ formesid += self.lems[lem].formes
+ else :
+ print('abscent : %s' % lem)
+ query = 'SELECT uces FROM uces where id IN %s ORDER BY id' % str(tuple(formesid))
+ res = self.cformes.execute(query)
+ return list(set(list(itertools.chain(*[[int(val) for val in row[0].split()] if not isinstance(row[0], int) else [row[0]] for row in res]))))
+
+ def gettgenstprof(self, tgen, classe, i, clnb):
+ tgenst = []
+ for lem in tgen :
+ if lem in self.lems :
+ lemst = self.getlemuces(lem)
+ tgenst += lemst
+ if not lem in self.tgenlem :
+ self.tgenlem[lem] = [0] * clnb
+ self.tgenlem[lem][i] = len(set(lemst).intersection(classe))
+ else :
+ print('abscent: ',lem)
+ return list(set(tgenst))
+
+ def gettgentxt(self, tgen):
+ sts = self.gettgenst(tgen)
+ return list(set([self.getucefromid(val).uci for val in sts]))
+
def getlemucis(self, lem) :
uces = self.getlemuces(lem)
return list(set([self.getucefromid(val).uci for val in uces]))
def getlemuceseff(self, lem, luces = None) :
- formesid = ', '.join([`val` for val in self.lems[lem].formes])
+ formesid = ', '.join([repr(val) for val in self.lems[lem].formes])
query = 'SELECT uces FROM uces where id IN (%s) ORDER BY id' % formesid
res = self.cformes.execute(query)
uces = list(itertools.chain(*[[int(val) for val in row[0].split()] if not isinstance(row[0], int) else [row[0]] for row in res]))
@@ -216,20 +258,25 @@ class Corpus :
def getucisize(self) :
ucesize = self.getucesize()
- return [sum(ucesize[uci.uces[0].ident:(uci.uces[-1].ident + 1)]) for uci in self.ucis]
-
+ return [sum(ucesize[uci.uces[0].ident:(uci.uces[-1].ident + 1)]) for uci in self.ucis if len(uci.uces) != 0]
+
def getucesize(self) :
res = self.getalluces()
return [len(uce[1].split()) for uce in res]
def getconcorde(self, uces) :
- return self.cuces.execute('select * from uces where id IN (%s) ORDER BY id;' % ', '.join([`i` for i in uces]))
-
+ return self.cuces.execute('select * from uces where id IN (%s) ORDER BY id;' % ', '.join([repr(i) for i in uces]))
+
def getuciconcorde(self, ucis) :
uces = [[val,[uce.ident for uce in self.ucis[val].uces]] for val in ucis]
uces = [[val[0], '\n'.join([row[1] for row in self.getconcorde(val[1])])] for val in uces]
return uces
+ def getuciconcorde_uces(self, uciid, uceid) :
+ uces = [uce.ident for uce in self.ucis[uciid].uces]
+ uces = [row for row in self.getconcorde(uces)]
+ return uces
+
def getwordconcorde(self, word) :
return self.getconcorde(self.getworduces(word))
@@ -238,14 +285,19 @@ class Corpus :
def getalluces(self) :
return self.cuces.execute('SELECT * FROM uces')
-
+
def getallucis(self):
uces = [row[1] for row in self.getalluces()]
return [[uci.ident, '\n'.join([uces[uce.ident] for uce in uci.uces])] for uci in self.ucis]
-
+
def getucesfrometoile(self, etoile) :
return [uce.ident for uci in self.ucis for uce in uci.uces if etoile in uci.etoiles]
+ def getucisfrometoile(self, etoile):
+ uces = [uce.ident for uci in self.ucis for uce in uci.uces if etoile in uci.etoiles]
+ return list(set([self.getucefromid(val).uci for val in uces]))
+
+
def getetoileuces(self) :
log.info('get uces etoiles')
etoileuces = {}
@@ -267,7 +319,7 @@ class Corpus :
else :
idpara += 1
return etoileuces
-
+
def getetoileucis(self):
etoileuces = {}
for uci in self.ucis :
@@ -288,6 +340,8 @@ class Corpus :
def getactivesnb(self, key) :
return len([lem for lem in self.lems if self.lems[lem].act == key])
+
+# fonction inactive mais avec une incertitude concernant l'indentation sur le dernier else
# def make_lems(self, lem = True) :
# log.info('make lems')
# self.lems = {}
@@ -314,7 +368,7 @@ class Corpus :
self.lems[self.formes[forme].lem] = Lem(self, self.formes[forme])
else :
self.lems = dict([[forme, Lem(self, self.formes[forme])] for forme in self.formes])
-
+
def make_lems_from_dict(self, dictionnaire, dolem = True) :
log.info('make lems from dict')
self.lems = {}
@@ -323,10 +377,10 @@ class Corpus :
lem = dictionnaire[forme][0]
gram = dictionnaire[forme][1]
elif forme.isdigit() :
- gram = u'num'
+ gram = 'num'
lem = forme
else :
- gram = u'nr'
+ gram = 'nr'
lem = forme
self.formes[forme].lem = lem
self.formes[forme].gram = gram
@@ -338,7 +392,7 @@ class Corpus :
self.lems[self.formes[forme].lem] = Lem(self, self.formes[forme])
else :
self.lems[forme] = Lem(self, self.formes[forme])
-
+
def make_idformes(self) :
self.idformes = dict([[self.formes[forme].ident, self.formes[forme]] for forme in self.formes])
@@ -363,11 +417,13 @@ class Corpus :
tab = []
for lem in tokeep :
deff = self.getlemuceseff(lem)
- ucesk = deff.keys()
- tab.append([lem] + [sum([deff[uce] for uce in et.intersection(ucesk)]) for et in etuces])
+ ucesk = list(deff.keys())
+ line = [lem] + [sum([deff[uce] for uce in et.intersection(ucesk)]) for et in etuces]
+ if sum(line[1:]) >= mineff :
+ tab.append(line)
tab.insert(0, [''] + etoiles)
return tab
-
+
def make_tgen_table(self, tgen, etoiles, tot = None):
lclasses = [self.getucesfrometoile(etoile) for etoile in etoiles]
sets = [set(cl) for cl in lclasses]
@@ -376,7 +432,7 @@ class Corpus :
for forme in self.formes :
formeuceeff = self.getformeuceseff(forme)
for i, classe in enumerate(lclasses) :
- concern = sets[i].intersection(formeuceeff.keys())
+ concern = sets[i].intersection(list(formeuceeff.keys()))
if len(concern) :
totoccurrences[etoiles[i]] += sum([formeuceeff[uce] for uce in concern])
#tgenoccurrences = dict([[val, 0] for val in etoiles])
@@ -386,11 +442,35 @@ class Corpus :
for lem in tgen[t] :
lemuceeff = self.getlemuceseff(lem)
for i, classe in enumerate(lclasses) :
- concern = sets[i].intersection(lemuceeff.keys())
+ concern = sets[i].intersection(list(lemuceeff.keys()))
if len(concern) :
tgenoccurrences[t][etoiles[i]] += sum([lemuceeff[uce] for uce in concern])
return tgenoccurrences, totoccurrences
-
+
+ def make_tgen_profile(self, tgen, ucecl, uci = False) :
+ log.info('tgen/classes')
+ self.tgenlem = {}
+ clnb = len(ucecl)
+ if uci :
+ #FIXME : NE MARCHE PLUS CHANGER CA
+ tab = [[lem] + [len(set(self.gettgentxt(tgen[lem])).intersection(classe)) for i, classe in enumerate(ucecl)] for lem in tgen]
+ else :
+ tab = [[lem] + [len(set(self.gettgenstprof(tgen[lem], classe, i, clnb)).intersection(classe)) for i, classe in enumerate(ucecl)] for lem in tgen]
+ tab = [[line[0]] + [val for val in line[1:]] for line in tab if sum(line[1:]) >= 3]
+ return tab
+
+ #i = 0
+ #nam = 'total'
+ #while nam + `i` in tgen :
+ # i += 1
+ #nam = nam + `i`
+ #last = [nam] + [`len(classe)` for classe in ucecl]
+ #tab += [last]
+ #line0 = ['tgen'] + ['_'.join(['cluster', `i+1`]) for i in range(len(ucecl))]
+ #tab = [line0] + tab
+ #with open(fileout, 'w') as f :
+ # f.write('\n'.join(['\t'.join(line) for line in tab]).encode(self.parametres['syscoding']))
+
def make_efftype_from_etoiles(self, etoiles) :
dtype = {}
etuces = [[] for et in etoiles]
@@ -403,7 +483,7 @@ class Corpus :
etuces = [set(val) for val in etuces]
for lem in self.lems :
deff = self.getlemuceseff(lem)
- ucesk = deff.keys()
+ ucesk = list(deff.keys())
gram = self.lems[lem].gram
if gram in dtype :
dtype[gram] = [i + j for i, j in zip(dtype[gram], [sum([deff[uce] for uce in et.intersection(ucesk)]) for et in etuces])]
@@ -456,8 +536,8 @@ class Corpus :
log.info('taille uc1 : %i - taille uc2 : %i' % (len(uc1), len(uc2)))
self.write_ucmatrix(uc1, actives, uc1out)
self.write_ucmatrix(uc2, actives, uc2out)
- listuce1 = [['uce', 'uc']] + [[`uce`, `i`] for i, ucl in enumerate(uc1) for uce in ucl]
- listuce2 = [['uce', 'uc']] + [[`uce`, `i`] for i, ucl in enumerate(uc2) for uce in ucl]
+ listuce1 = [['uce', 'uc']] + [[repr(uce), repr(i)] for i, ucl in enumerate(uc1) for uce in ucl]
+ listuce2 = [['uce', 'uc']] + [[repr(uce), repr(i)] for i, ucl in enumerate(uc2) for uce in ucl]
with open(listuce1out, 'w') as f :
f.write('\n'.join([';'.join(line) for line in listuce1]))
with open(listuce2out, 'w') as f :
@@ -474,7 +554,7 @@ class Corpus :
for uce in self.getlemuces(lem):
if (uces_uc[uce], i) not in deja_la :
nbl += 1
- f.write(''.join([' '.join([`uces_uc[uce]+1`,`i+1`,`1`]),'\n']))
+ f.write(''.join([' '.join([repr(uces_uc[uce]+1),repr(i+1),repr(1)]),'\n']))
deja_la[(uces_uc[uce], i)] = 0
f.seek(0)
with open(fileout, 'w') as ffin :
@@ -491,24 +571,32 @@ class Corpus :
self.make_iduces()
actuci = ''
actpara = False
- with open(outf,'w') as f :
+ with open(outf,'w', encoding='utf8') as f :
for uce in res :
if self.iduces[uce[0]].uci == actuci and self.iduces[uce[0]].para == actpara :
- f.write(uce[1].encode(self.parametres['syscoding']) + '\n')
+ f.write(uce[1] + '\n')
elif self.iduces[uce[0]].uci != actuci :
actuci = self.iduces[uce[0]].uci
if self.ucis[self.iduces[uce[0]].uci].paras == [] :
actpara = self.iduces[uce[0]].para
- f.write('\n' + ' '.join(self.ucis[self.iduces[uce[0]].uci].etoiles).encode(self.parametres['syscoding']) + '\n' + uce[1].encode(self.parametres['syscoding']) + '\n')
+ f.write('\n' + ' '.join(self.ucis[self.iduces[uce[0]].uci].etoiles) + '\n' + uce[1] + '\n')
else :
ident = 0
actpara = self.iduces[uce[0]].para
- f.write('\n'.join([' '.join(self.ucis[self.iduces[uce[0]].uci].etoiles).encode(self.parametres['syscoding']), self.ucis[self.iduces[uce[0]].uci].paras[ident].encode(self.parametres['syscoding']), uce[1].encode(self.parametres['syscoding'])]) + '\n')
+ f.write('\n'.join([' '.join(self.ucis[self.iduces[uce[0]].uci].etoiles), self.ucis[self.iduces[uce[0]].uci].paras[ident], uce[1]] + '\n'))
elif self.iduces[uce[0]].para != actpara :
actpara = self.iduces[uce[0]].para
ident += 1
- f.write('\n'.join([self.ucis[self.iduces[uce[0]].uci].paras[ident].encode(self.parametres['syscoding']), uce[1].encode(self.parametres['syscoding'])]) + '\n')
-
+ f.write('\n'.join([self.ucis[self.iduces[uce[0]].uci].paras[ident], uce[1]]) + '\n')
+
+ def export_meta_table(self, outf) :
+ metas = [[repr(i)] + text.etoiles[1:] for i, text in enumerate(self.ucis)]
+ longueur_max = max([len(val) for val in metas])
+ first = ['column_%i' % i for i in range(longueur_max)]
+ metas.insert(0, first)
+ with open(outf, 'w', encoding='utf8') as f :
+ f.write('\n'.join(['\t'.join(line) for line in metas]))
+
def export_corpus_classes(self, outf, alc = True, lem = False, uci = False) :
ucecl = {}
for i, lc in enumerate(self.lc) :
@@ -521,7 +609,7 @@ class Corpus :
self.make_iduces()
else :
res = self.getallucis()
- with open(outf, 'w') as f :
+ with open(outf, 'w', encoding='utf8') as f :
for uce in res :
guce = uce[1]
if not uci :
@@ -534,27 +622,27 @@ class Corpus :
etline = ' '.join(self.ucis[actuci].etoiles + ['*classe_%i' % ucecl[uce[0]]])
else :
etline = ' '.join(['<' + '='.join(et.split('_')) + '>' for et in self.ucis[actuci].etoiles[1:]])
- f.write(etline.encode(self.parametres['syscoding']) + '\n')
- f.write(guce.encode(self.parametres['syscoding']) + '\n\n')
+ f.write(etline + '\n')
+ f.write(guce + '\n\n')
def export_classe(self, outf, classe, lem = False, uci = False) :
- sts = self.lc[classe - 1]
+ sts = self.lc[classe - 1]
if not uci :
res = self.getconcorde(sts)
self.make_iduces()
else :
res = self.getuciconcorde(sts)
- with open(outf, 'w') as f :
+ with open(outf, 'w', encoding='utf8') as f :
for uce in res :
guce = uce[1]
if not uci :
- f.write(' '.join(self.ucis[self.iduces[uce[0]].uci].etoiles).encode(self.parametres['syscoding']) + '\n')
+ f.write(' '.join(self.ucis[self.iduces[uce[0]].uci].etoiles) + '\n')
else :
- f.write(' '.join(self.ucis[uce[0]].etoiles).encode(self.parametres['syscoding']) + '\n')
+ f.write(' '.join(self.ucis[uce[0]].etoiles) + '\n')
if lem :
guce = ' '.join([self.formes[forme].lem for forme in guce.split()])
- f.write(guce.encode(self.parametres['syscoding']) + '\n\n')
-
+ f.write(guce + '\n\n')
+
def export_owledge(self, rep, classe, lem = False, uci = False) :
sts = self.lc[classe - 1]
if not uci :
@@ -565,12 +653,12 @@ class Corpus :
for uce in res :
ident = uce[0]
guce = uce[1]
- outf = '.'.join([`ident`, 'txt'])
+ outf = '.'.join([repr(ident), 'txt'])
outf = os.path.join(rep, outf)
if lem :
guce = ' '.join([self.formes[forme].lem for forme in guce.split()])
- with open(outf, 'w') as f :
- f.write(guce.encode('cp1252', errors = 'replace'))
+ with open(outf, 'w', encoding='utf8') as f :
+ f.write(guce) #.encode('cp1252', errors = 'replace'))
def export_tropes(self, fileout, classe, lem = False, uci = False) :
sts = self.lc[classe - 1]
@@ -579,12 +667,12 @@ class Corpus :
self.make_iduces()
else :
res = self.getuciconcorde(sts)
- with open(fileout, 'w') as f :
+ with open(fileout, 'w', encoding='utf8') as f :
for uce in res :
guce = uce[1]
if lem :
guce = ' '.join([self.formes[forme].lem for forme in guce.split()])
- f.write(guce.encode('cp1252', errors = 'replace'))
+ f.write(guce) #.encode('cp1252', errors = 'replace'))
f.write('\n')
def make_and_write_sparse_matrix_from_uces(self, actives, outfile, listuce = False) :
@@ -594,16 +682,16 @@ class Corpus :
for i, lem in enumerate(actives) :
for uce in sorted(self.getlemuces(lem)) :
nbl += 1
- f.write(''.join([' '.join([`uce+1`, `i+1`,`1`]),'\n']))
+ f.write(''.join([' '.join([repr(uce+1), repr(i+1),repr(1)]),'\n']))
f.seek(0)
- with open(outfile, 'w') as ffin :
+ with open(outfile, 'w') as ffin :
ffin.write("%%%%MatrixMarket matrix coordinate integer general\n%i %i %i\n" % (self.getucenb(), len(actives), nbl))
for line in f :
ffin.write(line)
os.remove(outfile + '~')
if listuce :
with open(listuce, 'w') as f :
- f.write('\n'.join(['uce;uc'] + [';'.join([`i`,`i`]) for i in range(0, self.getucenb())]))
+ f.write('\n'.join(['uce;uc'] + [';'.join([repr(i),repr(i)]) for i in range(0, self.getucenb())]))
def make_and_write_sparse_matrix_from_uci(self, actives, outfile, listuci = False) :
log.info('make_and_write_sparse_matrix_from_ucis %s' % outfile)
@@ -612,17 +700,17 @@ class Corpus :
for i, lem in enumerate(actives) :
for uci in sorted(self.getlemucis(lem)) :
nbl += 1
- f.write(''.join([' '.join([`uci+1`, `i+1`,`1`]),'\n']))
+ f.write(''.join([' '.join([repr(uci+1), repr(i+1),repr(1)]),'\n']))
f.seek(0)
- with open(outfile, 'w') as ffin :
+ with open(outfile, 'w') as ffin :
ffin.write("%%%%MatrixMarket matrix coordinate integer general\n%i %i %i\n" % (self.getucinb(), len(actives), nbl))
for line in f :
ffin.write(line)
os.remove(outfile + '~')
if listuci :
with open(listuci, 'w') as f :
- f.write('\n'.join(['uci;uc'] + [';'.join([`i`,`i`]) for i in range(0, self.getucinb())]))
-
+ f.write('\n'.join(['uci;uc'] + [';'.join([repr(i),repr(i)]) for i in range(0, self.getucinb())]))
+
def make_and_write_sparse_matrix_from_classe(self, actives, uces, outfile) :
log.info('make_and_write_sparse_matrix_from_classe %s' % outfile)
nbl = 0
@@ -631,14 +719,14 @@ class Corpus :
for i, lem in enumerate(actives) :
uces_ok = list(set(self.getlemuces(lem)).intersection(uces))
for uce in uces_ok :
- f.write(''.join([' '.join([`duces[uce]+1`,`i+1`,`1`]),'\n']))
+ f.write(''.join([' '.join([repr(duces[uce]+1),repr(i+1),repr(1)]),'\n']))
f.seek(0)
- with open(outfile, 'w') as ffin :
- ffin.write("%%%%MatrixMarket matrix coordinate integer general\n%i %i %i\n" % (self.getucenb(), len(actives), nbl))
+ with open(outfile, 'w') as ffin :
+ ffin.write("%%%%MatrixMarket matrix coordinate integer general\n%i %i %i\n" % (len(uces), len(actives), nbl))
for line in f :
ffin.write(line)
os.remove(outfile + '~')
-
+
def make_table_with_classe(self, uces, list_act, uci = False) :
table_uce = [[0 for val in list_act] for line in range(0,len(uces))]
uces = dict([[uce, i] for i, uce in enumerate(uces)])
@@ -651,8 +739,8 @@ class Corpus :
for uce in lemuces :
table_uce[uces[uce]][i] = 1
table_uce.insert(0, list_act)
- return table_uce
-
+ return table_uce
+
def make_pondtable_with_classe(self, uces, list_act) :
table_uce = [[0 for val in list_act] for line in range(0,len(uces))]
uces = dict([[uce, i] for i, uce in enumerate(uces)])
@@ -662,7 +750,7 @@ class Corpus :
for uce in lemuces :
table_uce[uces[uce]][i] = uceseff[uce]
table_uce.insert(0, list_act)
- return table_uce
+ return table_uce
def parse_active(self, gramact, gramsup = None) :
log.info('parse actives')
@@ -683,7 +771,7 @@ class Corpus :
if self.idformes is None :
self.make_idformes()
return [lem for lem in self.lems if self.getlemeff(lem) >= limit and self.lems[lem].act == key]
-
+
def make_actives_nb(self, nbmax, key) :
log.info('make_actives_nb : %i - %i' % (nbmax,key))
if self.idformes is None :
@@ -711,7 +799,7 @@ class Corpus :
stop = nbmax - 1
lim = effs[stop]
log.info('nb actives = %i - eff min = %i ' % (stop + 1, lim))
- return [val[1] for val in allactives[0:stop + 1]], lim
+ return [val[1] for val in allactives[0:stop]], lim
def make_and_write_profile(self, actives, ucecl, fileout, uci = False) :
log.info('formes/classes')
@@ -719,16 +807,16 @@ class Corpus :
tab = [[lem] + [len(set(self.getlemucis(lem)).intersection(classe)) for classe in ucecl] for lem in actives]
else :
tab = [[lem] + [len(set(self.getlemuces(lem)).intersection(classe)) for classe in ucecl] for lem in actives]
- tab = [[line[0]] + [`val` for val in line[1:]] for line in tab if sum(line[1:]) >= 3]
- with open(fileout, 'w') as f :
- f.write('\n'.join([';'.join(line) for line in tab]).encode(self.parametres['syscoding']))
+ tab = [[line[0]] + [repr(val) for val in line[1:]] for line in tab if sum(line[1:]) >= 3]
+ with open(fileout, 'w', encoding='utf8') as f :
+ f.write('\n'.join([';'.join(line) for line in tab]))
def make_etoiles(self) :
etoiles = set([])
for uci in self.ucis :
etoiles.update(uci.etoiles[1:])
return list(etoiles)
-
+
def make_themes(self):
themes = set([])
for uci in self.ucis :
@@ -757,6 +845,28 @@ class Corpus :
det[et[0]] = 1
return det
+ def make_theme_dict(self):
+ themes = [val for uci in self.ucis for val in uci.paras]
+ det = {}
+ for theme in themes :
+ th = theme.split('_')
+ if th[0] in det :
+ try :
+ endth = '_'.join(th[1:])
+ if theme in det[th[0]] :
+ det[th[0]][theme] += 1
+ else :
+ det[th[0]][theme] = 1
+ except IndexError :
+ det[th[0]] += 1
+ else :
+ try :
+ endth = '_'.join(th[1:])
+ det[th[0]] = {theme:1}
+ except IndexError :
+ det[th[0]] = 1
+ return det
+
def make_etline(self, listet) :
etuces = [[] for et in listet]
for uci in self.ucis :
@@ -773,9 +883,9 @@ class Corpus :
etoileuces = self.getetoileuces()
else :
etoileuces = self.getetoileucis()
- etoileuces = dict([[et, etoileuces[et]] for et in etoileuces if len(etoileuces[et]) > 1])
- with open(fileout, 'w') as f :
- f.write('\n'.join([';'.join([et] + [`len(set(etoileuces[et]).intersection(classe))` for classe in ucecl]) for et in etoileuces]).encode(self.parametres['syscoding']))
+ etoileuces = dict([[et, etoileuces[et]] for et in etoileuces if len(etoileuces[et]) > 0])
+ with open(fileout, 'w', encoding='utf8') as f :
+ f.write('\n'.join([';'.join([et] + [repr(len(set(etoileuces[et]).intersection(classe))) for classe in ucecl]) for et in etoileuces])) #.encode(self.parametres['syscoding'])
#etoiles = self.make_etoiles()
#with open(fileout, 'w') as f :
# f.write('\n'.join([';'.join([etoile] + [`len(set(self.getucesfrometoile(etoile)).intersection(classe))` for classe in ucecl]) for etoile in etoiles]).encode(self.parametres['syscoding']))
@@ -783,15 +893,15 @@ class Corpus :
def make_colored_corpus(self, uci = False) :
ucecl = {}
for i, lc in enumerate(self.lc) :
- for uce in lc :
+ for uce in lc :
ucecl[uce] = i + 1
for uce in self.lc0 :
ucecl[uce] = 0
- color = ['black'] + colors[len(self.lc) - 1]
+ color = ['black'] + colors[len(self.lc) - 1]
txt = '''
-
+
-''' % sys.getdefaultencoding()
+'''
if not uci :
res = self.getalluces()
self.make_iduces()
@@ -816,6 +926,32 @@ class Corpus :
txt += '' % (color[ucecl[uce[0]]]) + uce[1] + '
'
return txt + '\n'
+ def make_cut_corpus(self, uci = False) :
+ txt = ''
+ if not uci :
+ res = self.getalluces()
+ self.make_iduces()
+ actuci = ''
+ actpara = False
+ for uce in res :
+ if self.iduces[uce[0]].uci != actuci :
+ actuci = self.iduces[uce[0]].uci
+ txt += '\n' + ' '.join(self.ucis[self.iduces[uce[0]].uci].etoiles) + '\n'
+ txt += ''.join(['\n',uce[1],'\n'])
+ else :
+ txt += ''.join(['\n',uce[1],'\n'])
+ else :
+ res = self.getallucis()
+ actuci = ''
+ for uce in res :
+ if self.ucis[uce[0]].ident != actuci :
+ actuci = self.ucis[uce[0]].ident
+ txt += '\n' + ' '.join(self.ucis[self.ucis[uce[0]].ident].etoiles) + '\n'
+ txt += ''.join(['\n',uce[1],'\n'])
+ else :
+ txt += ''.join(['\n',uce[1],'\n'])
+ return txt
+
def count_from_list(self, l, d) :
for val in l :
if val in d :
@@ -847,7 +983,7 @@ class Corpus :
def find_segments_in_classe(self, list_uce, taille_segment, taille_limite, uci = False):
d={}
- if not uci :
+ if not uci :
concorde = self.getconcorde
else :
concorde = self.getuciconcorde
@@ -860,7 +996,7 @@ class Corpus :
if len(l) > taille_limite :
l = l[-taille_limite:]
return l
-
+
def make_segments_profile(self, fileout, lenmin = 3, lenmax = 10, effmin = 50, lem = False) :
d = {}
for b, classe in enumerate(self.lc) :
@@ -871,9 +1007,9 @@ class Corpus :
for taille_segment in range(lenmin,lenmax) :
d =self.count_from_list_cl([' '.join(uce[i:i+taille_segment]) for i in range(len(uce)-(taille_segment - 1))], d, b, len(self.lc))
result = [[seg] + [str(val) for val in d[seg]] for seg in d if sum(d[seg]) >= effmin]
- with open(fileout, 'w') as f :
+ with open(fileout, 'w', encoding='utf8') as f :
f.write('\n'.join([';'.join(line) for line in result]))
-
+
def make_proftype(self, outf) :
res = {}
for lem in self.lems :
@@ -882,16 +1018,15 @@ class Corpus :
res[gram] = [0 for val in self.lc]
lemuceeff = self.getlemuceseff(lem)
for i, classe in enumerate(self.lc) :
- concern = set(classe).intersection(lemuceeff.keys())
+ concern = set(classe).intersection(list(lemuceeff.keys()))
res[gram][i] += sum([lemuceeff[uce] for uce in concern])
- res = [[gram] + [`val` for val in res[gram]] for gram in res]
+ res = [[gram] + [repr(val) for val in res[gram]] for gram in res]
res.sort()
- with open(outf, 'w') as f :
- f.write('\n'.join([';'.join(line) for line in res]).encode(self.parametres['syscoding']))
-
+ with open(outf, 'w', encoding='utf8') as f :
+ f.write('\n'.join([';'.join(line) for line in res]))
def make_ucecl_from_R(self, filein) :
- with open(filein, 'rU') as f :
+ with open(filein, 'r') as f :
c = f.readlines()
c.pop(0)
self.lc = []
@@ -904,7 +1039,7 @@ class Corpus :
self.lc = [[uce[0] for uce in self.lc if uce[1] == i] for i in range(clnb+1)]
self.lc0 = self.lc.pop(0)
#return ucecl
-
+
def get_stat_by_cluster(self, outf, lclasses = None) :
log.info('get_stat_by_cluster')
if lclasses is None :
@@ -918,19 +1053,19 @@ class Corpus :
for forme in self.formes :
formeuceeff = self.getformeuceseff(forme)
for i, classe in enumerate(lclasses) :
- concern = sets[i].intersection(formeuceeff.keys())
+ concern = sets[i].intersection(list(formeuceeff.keys()))
if len(concern) :
occurrences[i+1] += sum([formeuceeff[uce] for uce in concern])
formescl[i+1] += 1
if self.formes[forme].freq == 1 :
hapaxcl[i+1] += 1
- log.info('%f' % (time() - t1))
+ log.info('%f' % (time() - t1))
if outf is not None :
- toprint = '\n'.join([';'.join([`i`, `occurrences[i]`, `formescl[i]`, `hapaxcl[i]`, `lenclasses[i]`, `float(hapaxcl[i])/float(formescl[i])`]) for i in occurrences])
- with open(outf, 'w') as f :
+ toprint = '\n'.join([';'.join([repr(i), repr(occurrences[i]), repr(formescl[i]), repr(hapaxcl[i]), repr(lenclasses[i]), repr(float(hapaxcl[i])/float(formescl[i]))]) for i in occurrences])
+ with open(outf, 'w', encoding='utf8') as f :
f.write(toprint)
else :
- return [[`occurrences[i]`, `formescl[i]`, `hapaxcl[i]`, `lenclasses[i]`, `float(hapaxcl[i])/float(formescl[i])`] for i in occurrences]
+ return [[repr(occurrences[i]), repr(formescl[i]), repr(hapaxcl[i]), repr(lenclasses[i]), repr(float(hapaxcl[i])/float(formescl[i]))] for i in occurrences]
def get_stat_by_et(self, outf, etoiles) :
lclasses = [self.getucesfrometoile(etoile) for etoile in etoiles]
@@ -971,7 +1106,7 @@ class Corpus :
huces[hucesdict[uce][0]].append(uce)
else :
huces[hucesdict[uce][0]] = [uce]
- huces = zip(huces, huces.values())
+ huces = list(zip(huces, list(huces.values())))
huces.sort(reverse=True)
txt = """
@@ -991,23 +1126,22 @@ class Corpus :
txt += """
"""
- with open('/tmp/testhapxuce.html','w') as f :
+ with open('/tmp/testhapxuce.html','w', encoding='utf8') as f :
f.write(txt)
def export_dictionary(self, fileout, syscoding) :
listformes = [[self.formes[forme].freq, forme, self.formes[forme].lem, self.formes[forme].gram] for forme in self.formes]
listformes.sort(reverse = True)
- listformes = [forme[1:] + [`forme[0]`] for forme in listformes]
- with open(fileout, 'w') as f :
- f.write('\n'.join(['\t'.join(forme) for forme in listformes]).encode(syscoding))
+ listformes = [forme[1:] + [repr(forme[0])] for forme in listformes]
+ with open(fileout, 'w', encoding='utf8') as f :
+ f.write('\n'.join(['\t'.join(forme) for forme in listformes]))
def export_lems(self, fileout, syscoding) :
self.make_idformes()
- listlem = [[lem, '\t'.join(['\t'.join([self.idformes[forme].forme, `self.lems[lem].formes[forme]`]) for forme in self.lems[lem].formes])] for lem in self.lems]
+ listlem = [[lem, '\t'.join(['\t'.join([self.idformes[forme].forme, repr(self.lems[lem].formes[forme])]) for forme in self.lems[lem].formes])] for lem in self.lems]
listlem.sort()
- with open(fileout, 'w') as f :
- f.write('\n'.join(['\t'.join(lem) for lem in listlem]).encode(syscoding))
-
+ with open(fileout, 'w', encoding='utf8') as f :
+ f.write('\n'.join(['\t'.join(lem) for lem in listlem]))
class MakeUciStat :
@@ -1015,7 +1149,7 @@ class MakeUciStat :
ucinb = corpus.getucinb()
ucisize = corpus.getucisize()
ucimean = float(sum(ucisize))/float(ucinb)
- detoile = corpus.make_etoiles_dict()
+ detoile = corpus.make_etoiles_dict()
class Uci :
def __init__(self, iduci, line, paraset = None) :
@@ -1056,24 +1190,23 @@ class Lem :
self.formes[forme.ident] = forme.freq
self.freq += forme.freq
+
def decouperlist(chaine, longueur, longueurOptimale) :
"""
on part du dernier caractère, et on recule jusqu'au début de la chaîne.
Si on trouve un '$', c'est fini.
Sinon, on cherche le meilleur candidat. C'est-Ã -dire le rapport poids/distance le plus important.
"""
- separateurs = [[u'.', 6.0], [u'?', 6.0], [u'!', 6.0], [u'£$£', 6.0], [u':', 5.0], [u';', 4.0], [u',', 1.0], [u' ', 0.01]]
+ separateurs = [['.', 6.0], ['?', 6.0], ['!', 6.0], ['£$£', 6.0], [':', 5.0], [';', 4.0], [',', 1.0], [' ', 0.01]]
dsep = dict([[val[0],val[1]] for val in separateurs])
trouve = False # si on a trouvé un bon séparateur
iDecoupe = 0 # indice du caractere ou il faut decouper
-
longueur = min(longueur, len(chaine) - 1)
chaineTravail = chaine[:longueur + 1]
nbCar = longueur
meilleur = ['', 0, 0] # type, poids et position du meilleur separateur
-
try :
- indice = chaineTravail.index(u'$')
+ indice = chaineTravail.index('$')
trouve = True
iDecoupe = indice - 1
except ValueError :
@@ -1100,10 +1233,12 @@ def decouperlist(chaine, longueur, longueurOptimale) :
nbCar = nbCar - 1
# si on a trouvé
if trouve:
+
#if meilleur[0] != ' ' :
# fin = chaine[iDecoupe + 1:]
# retour = chaineTravail[:iDecoupe]
#else :
+
fin = chaine[iDecoupe + 1:]
retour = chaineTravail[:iDecoupe + 1]
return len(retour) > 0, retour, fin
@@ -1111,21 +1246,23 @@ def decouperlist(chaine, longueur, longueurOptimale) :
return False, chaine, ''
def testetoile(line) :
- return line.startswith(u'****')
+ return line.startswith('****')
def testint(line) :
- return line[0:4].isdigit() and u'*' in line
+ return line[0:4].isdigit() and '*' in line
def prep_txtlist(txt) :
- return txt.split() + [u'$']
+ return txt.split() + ['$']
def prep_txtcharact(txt) :
- return txt + u'$'
+ return txt + '$'
+
class BuildCorpus :
"""
Class for building a corpus
"""
+
def __init__(self, infile, parametres_corpus, lexique = None, expressions = None, dlg = None) :
log.info('begin building corpus...')
self.lexique = lexique
@@ -1139,12 +1276,12 @@ class BuildCorpus :
self.corpus.pathout = PathOut(filename = parametres_corpus['originalpath'], dirout = parametres_corpus['pathout'])
self.corpus.pathout.createdir(parametres_corpus['pathout'])
self.corpus.parametres['uuid'] = str(uuid4())
- self.corpus.parametres['corpus_name'] = parametres_corpus['corpus_name']#os.path.split(self.corpus.parametres['pathout'])[1]
+ self.corpus.parametres['corpus_name'] = parametres_corpus['corpus_name'] #os.path.split(self.corpus.parametres['pathout'])[1]
self.corpus.parametres['type'] = 'corpus'
if self.corpus.parametres['keep_ponct'] :
self.ponctuation_espace = [' ', '']
else :
- self.ponctuation_espace = [' ','.', u'£$£', ';', '?', '!', ',', ':','']
+ self.ponctuation_espace = [' ','.', '£$£', ';', '?', '!', ',', ':','']
self.cleans = []
self.tolist = self.corpus.parametres.get('tolist', 0)
self.buildcleans()
@@ -1165,14 +1302,14 @@ class BuildCorpus :
self.ucesize = self.corpus.parametres.get('ucesize', 240)
log.info('method uce : %s' % method)
- def dobuild(self) :
+ def dobuild(self) :
t1 = time()
try :
self.read_corpus(self.infile)
- except Warning, args :
+ except Warning as args :
log.info('pas kool %s' % args)
raise Warning
- else :
+ else :
self.indexdb()
self.corpus.parametres['ira'] = self.corpus.pathout['Corpus.cira']
self.time = time() - t1
@@ -1233,7 +1370,7 @@ class BuildCorpus :
if self.corpus.parametres.get('firstclean', 1) :
self.cleans.append(self.firstclean)
if self.corpus.parametres['charact'] :
- self.rule = self.corpus.parametres.get('keep_caract', u"^a-zA-Z0-9à ÃâÃäÃáÃéÃèÃêÃëÃìÃîÃïÃòÃôÃöÃùÃûÃüÃçÃÃÅÅâñ.:,;!?*'_")
+ self.rule = self.corpus.parametres.get('keep_caract', "^a-zA-Z0-9à ÃâÃäÃáÃéÃèÃêÃëÃìÃîÃïÃòÃôÃöÃùÃûÃüÃçÃÃÅÅâñ.:,;!?*'_")
self.cleans.append(self.docharact)
if self.corpus.parametres.get('expressions', 1) :
self.cleans.append(self.make_expression)
@@ -1243,29 +1380,31 @@ class BuildCorpus :
self.cleans.append(self.dotiret)
def make_expression(self,txt) :
- for expression in self.expressions:
+ exp = list(self.expressions.keys())
+ exp.sort(reverse=True)
+ for expression in exp :
if expression in txt :
txt = txt.replace(expression, self.expressions[expression][0])
return txt
-
+
def dolower(self, txt) :
return txt.lower()
def docharact(self, txt) :
#rule = u"^a-zA-Z0-9à ÃâÃäÃáÃéÃèÃêÃëÃìÃîÃïÃòÃôÃöÃùÃûÃüÃçÃÃÅÅâñ.:,;!?*'_-"
- list_keep = u"[" + self.rule + "]+"
+ list_keep = "[" + self.rule + "]+"
return re.sub(list_keep, ' ', txt)
-
+
def doapos(self, txt) :
- return txt.replace(u'\'', u' ')
+ return txt.replace('\'', ' ')
def dotiret(self, txt) :
- return txt.replace(u'-', u' ')
+ return txt.replace('-', ' ')
def firstclean(self, txt) :
- txt = txt.replace(u'â',"'")
- txt = txt.replace(u'Å', u'oe')
- return txt.replace('...',u' £$£ ').replace('?',' ? ').replace('.',' . ').replace('!', ' ! ').replace(',',' , ').replace(';', ' ; ').replace(':',' : ').replace(u'â¦', u' £$£ ')
+ txt = txt.replace('â',"'")
+ txt = txt.replace('Å', 'oe')
+ return txt.replace('...',' £$£ ').replace('?',' ? ').replace('.',' . ').replace('!', ' ! ').replace(',',' , ').replace(';', ' ; ').replace(':',' : ').replace('â¦', ' £$£ ')
def make_cleans(self, txt) :
for clean in self.cleans :
@@ -1275,11 +1414,11 @@ class BuildCorpus :
def backup_uce(self) :
if self.corpus.idformesuces != {} :
log.info('backup %i' % len(self.corpus.idformesuces))
- touce = [(`forme`, ' '.join([`val` for val in self.corpus.idformesuces[forme].keys()])) for forme in self.corpus.idformesuces]
- toeff = [(`forme`, ' '.join([`val` for val in self.corpus.idformesuces[forme].values()])) for forme in self.corpus.idformesuces]
+ touce = [(repr(forme), ' '.join([repr(val) for val in list(self.corpus.idformesuces[forme].keys())])) for forme in self.corpus.idformesuces]
+ toeff = [(repr(forme), ' '.join([repr(val) for val in list(self.corpus.idformesuces[forme].values())])) for forme in self.corpus.idformesuces]
self.cf.executemany('INSERT INTO uces VALUES (?,?);', touce)
self.cf.executemany('INSERT INTO eff VALUES (?,?);', toeff)
- self.corpus.idformesuces = {}
+ self.corpus.idformesuces = {}
self.count = 1
def backup_corpus(self) :
@@ -1287,10 +1426,10 @@ class BuildCorpus :
t = time()
for uci in self.corpus.ucis :
self.ccorpus.execute('INSERT INTO etoiles VALUES (?,?,?);' ,(uci.ident,' '.join(uci.etoiles), ' '.join(uci.paras,)))
- for uce in uci.uces :
- self.ccorpus.execute('INSERT INTO luces VALUES (?,?,?);',(`uci.ident`,`uce.para`,`uce.ident`,))
+ for uce in uci.uces :
+ self.ccorpus.execute('INSERT INTO luces VALUES (?,?,?);',(repr(uci.ident),repr(uce.para),repr(uce.ident),))
for forme in self.corpus.formes :
- self.ccorpus.execute('INSERT INTO formes VALUES (?,?,?,?,?);', (`self.corpus.formes[forme].ident`, forme, self.corpus.formes[forme].lem, self.corpus.formes[forme].gram, `self.corpus.formes[forme].freq`,))
+ self.ccorpus.execute('INSERT INTO formes VALUES (?,?,?,?,?);', (repr(self.corpus.formes[forme].ident), forme, self.corpus.formes[forme].lem, self.corpus.formes[forme].gram, repr(self.corpus.formes[forme].freq),))
log.info('%f' % (time() - t))
def dofinish(self) :
@@ -1308,6 +1447,7 @@ class BuildCorpus :
self.corpus.parametres['hapax'] = '%i - %.2f %% des formes - %.2f %% des occurrences' % (hapaxnb, pourhapaxf, pourhapaxocc)
class BuildSubCorpus(BuildCorpus):
+
def __init__(self, corpus, parametres, dlg = None) :
log.info('begin subcorpus...')
self.dlg = dlg
@@ -1324,10 +1464,10 @@ class BuildSubCorpus(BuildCorpus):
self.corpus.parametres['meta'] = parametres.get('meta', False)
self.corpus.parametres['uuid'] = str(uuid4())
if parametres.get('frommeta', False) :
- print 'make subtexts'
+ print('make subtexts')
self.corpus.ucis = [CopyUci(uci) for uci in self.ori.ucis if set(parametres['meta']).intersection(uci.etoiles) != set()]
elif parametres.get('fromtheme', False) :
- print 'make subtexts from theme'
+ print('make subtexts from theme')
idpara = 0
for uci in self.ori.ucis :
if uci.paras != [] :
@@ -1344,7 +1484,7 @@ class BuildSubCorpus(BuildCorpus):
nuci.paras = newpara
self.corpus.ucis.append(nuci)
else :
- idpara += 1
+ idpara += 1
elif parametres.get('fromclusters', False) :
self.parametres['uceids'] = [st for i in self.parametres['meta'] for st in self.parametres['lc'][i]]
self.fromuceids()
@@ -1355,8 +1495,8 @@ class BuildSubCorpus(BuildCorpus):
self.dobuild()
def fromuceids(self):
- print 'fromuceids'
- dictucekeep = dict(zip(self.parametres['uceids'], self.parametres['uceids']))
+ print('fromuceids')
+ dictucekeep = dict(list(zip(self.parametres['uceids'], self.parametres['uceids'])))
idpara = 0
for uci in self.ori.ucis :
if uci.paras == [] :
@@ -1370,7 +1510,7 @@ class BuildSubCorpus(BuildCorpus):
newuces = []
newpara = []
for et in uci.paras :
- keepuces = [CopyUce(uce) for uce in uci.uces if uce.ident in dictucekeep]
+ keepuces = [CopyUce(uce) for uce in uci.uces if uce.ident in dictucekeepand and uce.para == idpara]
idpara += 1
if keepuces != [] :
newuces += keepuces
@@ -1379,8 +1519,8 @@ class BuildSubCorpus(BuildCorpus):
nuci = CopyUci(uci)
nuci.uces = newuces
nuci.paras = newpara
- self.corpus.ucis.append(nuci)
-
+ self.corpus.ucis.append(nuci)
+
def read_corpus(self, infile = None):
self.olduceid = [uce.ident for uci in self.corpus.ucis for uce in uci.uces]
ident_uci = 0
@@ -1388,7 +1528,7 @@ class BuildSubCorpus(BuildCorpus):
ident_para = -1
lastpara = -1
newuceident = {}
- print 'redo text, para and st ident'
+ print('redo text, para and st ident')
for uci in self.corpus.ucis :
uci.ident = ident_uci
ident_uci += 1
@@ -1403,16 +1543,18 @@ class BuildSubCorpus(BuildCorpus):
newuceident[uce.ident] = ident_uce
uce.ident = ident_uce
ident_uce += 1
- print 'backup st text and forms'
+ print('backup st text and forms')
for row in self.ori.getconcorde(self.olduceid) :
- self.c.execute('INSERT INTO uces VALUES(?,?);', (`newuceident[row[0]]`, row[1]))
+ self.c.execute('INSERT INTO uces VALUES(?,?);', (repr(newuceident[row[0]]), row[1]))
for word in row[1].split() :
self.corpus.add_word_from_forme(self.ori.formes[word], newuceident[row[0]])
self.backup_uce()
- print 'done'
+ print('done')
class BuildFromAlceste(BuildCorpus) :
+
def read_corpus(self, infile) :
+
if self.dlg is not None :
self.dlg.Pulse('textes : 0 - segments : 0')
self.limitshow = 0
@@ -1432,20 +1574,22 @@ class BuildFromAlceste(BuildCorpus) :
if self.testuci(line) :
iduci += 1
if txt != [] :
+ #doc = nlp(' '.join(txt))
+ #print([[word, word.pos_, word.lemma_] for word in doc])
iduce, idpara = self.treattxt(txt, iduce, idpara, iduci - 1)
txt = []
self.corpus.ucis.append(Uci(iduci, line))
else :
if iduci > 0 :
if self.corpus.ucis[-1].uces == [] :
- log.info(u'Empty text : %i' % linenb)
+ log.info('Empty text : %i' % linenb)
iduci -= 1
self.corpus.ucis.pop()
self.corpus.ucis.append(Uci(iduci, line))
if self.dlg is not None :
if not (iduci + 1) % 10 :
self.dlg.Pulse('textes : %i - segments : %i' % (iduci + 1, iduce +1))
- elif line.startswith(u'-*') :
+ elif line.startswith('-*') :
if iduci != -1 :
if txt != [] :
iduce, idpara = self.treattxt(txt, iduce, idpara, iduci)
@@ -1468,8 +1612,8 @@ class BuildFromAlceste(BuildCorpus) :
raise Exception('EmptyText %i' % linenb)
if iduci != -1 and iduce != -1:
self.backup_uce()
- else :
- log.info(_(u"No Text in corpus. Are you sure of the formatting ?"))
+ else :
+ log.info(_("No Text in corpus. Are you sure of the formatting ?"))
raise Exception('TextBeforeTextMark %i' % linenb)
except UnicodeDecodeError :
raise Exception("CorpusEncoding")
@@ -1489,7 +1633,7 @@ class BuildFromAlceste(BuildCorpus) :
for uce in ucetxt :
iduce += 1
self.corpus.ucis[-1].uces.append(Uce(iduce, idpara, iduci))
- self.c.execute('INSERT INTO uces VALUES(?,?);', (`iduce`,uce))
+ self.c.execute('INSERT INTO uces VALUES(?,?);', (repr(iduce),uce))
if not self.tolist :
uce = uce.split()
else :
@@ -1497,7 +1641,7 @@ class BuildFromAlceste(BuildCorpus) :
for word in uce :
self.last += 1
self.corpus.add_word(word)
- log.debug(' '.join([`iduci`,`idpara`,`iduce`]))
+ log.debug(' '.join([repr(iduci),repr(idpara),repr(iduce)]))
if self.last > self.lim :
self.backup_uce()
self.last = 0
@@ -1514,7 +1658,7 @@ class BuildFromAlceste(BuildCorpus) :
out.append(uce)
reste, texte_uce, suite = self.decouper(suite, self.ucesize + 15, self.ucesize)
uce = ' '.join([val for val in texte_uce if val not in self.ponctuation_espace])
- if uce != '' :
+ if uce != '' :
out.append(uce)
return out
else :
@@ -1526,6 +1670,7 @@ class BuildFromAlceste(BuildCorpus) :
#read (treat_txt)
class Builder :
+
def __init__(self, parent, dlg = None) :
self.parent = parent
self.dlg = dlg
@@ -1537,6 +1682,8 @@ class Builder :
dial.txtpath.SetLabel(parent.filename)
#dial.repout_choices.SetValue(parametres['pathout'])
self.res = dial.ShowModal()
+ if self.dlg is not None :
+ self.dlg = progressbar(self.parent, self.dlg)
if self.res == 5100 :
parametres = dial.doparametres()
parametres['originalpath'] = parent.filename
@@ -1545,6 +1692,9 @@ class Builder :
filein = parametres['dictionary']
else :
filein = None
+ if dial.corpusname.GetValue() != '' :
+ parametres['corpus_name'] = dial.corpusname.GetValue()
+ dial.Destroy()
ReadLexique(self.parent, lang = parametres['lang'], filein = filein)
if parametres['lang'] != 'other' and os.path.exists(self.parent.DictPath.get(parametres['lang']+'_exp', 'french_exp')):
self.parent.expressions = ReadDicoAsDico(self.parent.DictPath.get(parametres['lang']+'_exp', 'french_exp'))
@@ -1552,21 +1702,22 @@ class Builder :
self.parent.expressions = {}
self.parametres = parametres
else :
+ dial.Destroy()
if self.dlg is not None :
self.dlg.Destroy()
- dial.Destroy()
def doanalyse(self) :
return BuildFromAlceste(self.parent.filename, self.parametres, self.parent.lexique, self.parent.expressions, dlg = self.dlg).corpus
class SubBuilder :
+
def __init__(self, parent, corpus, parametres = None, dlg = None):
self.parent = parent
self.ori = corpus
self.dlg = dlg
corpus_name = 'Sub' + corpus.parametres['corpus_name']
if dlg is not None :
- busy = wx.BusyInfo(_("Please wait...").decode('utf8'), self)
+ busy = wx.BusyInfo(_("Please wait..."), self)
wx.SafeYield()
parametres['corpus_name'] = corpus_name
if parametres.get('frommeta', False) :
@@ -1574,7 +1725,7 @@ class SubBuilder :
elif parametres.get('fromtheme', False) :
parametres['meta'] = corpus.make_themes()
elif parametres.get('fromclusters', False) :
- parametres['meta'] = [' '.join(['classe', `i`]) for i in range(1,parametres['clnb'] + 1)]
+ parametres['meta'] = [' '.join(['classe', repr(i)]) for i in range(1,parametres['clnb'] + 1)]
else :
parametres['meta'] = []
if 'fromclusters' not in parametres :
@@ -1604,6 +1755,184 @@ class SubBuilder :
dial.Destroy()
else :
dial.Destroy()
-
+
def doanalyse(self):
return BuildSubCorpus(self.ori, parametres = self.parametres, dlg = self.dlg).corpus
+
+class BuildMergeFromClusters(BuildCorpus):
+
+ def __init__(self, analyses, parametres, dlg = None) :
+ log.info('begin subcorpus...')
+ self.dlg = dlg
+ self.infile = None
+ self.corpus = Corpus(self, {'type' : 'corpus', 'originalpath' : 'MergeFromClusters', 'encoding' : 'merge'})
+ self.last = 0
+ self.analyses = analyses
+ self.lcl = []
+ self.parametres = parametres
+ #self.encoding = corpus.parametres['encoding']
+ self.corpus.parametres['corpus_name'] = parametres['corpus_name']
+ self.corpus.pathout = PathOut(filename = 'MFC', dirout = parametres['pathout'])
+ self.corpus.pathout.createdir(parametres['pathout'])
+ self.corpus.parametres['pathout'] = parametres['pathout']
+ self.corpus.parametres['meta'] = parametres.get('meta', False)
+ self.corpus.parametres['uuid'] = str(uuid4())
+ for i, analyse in enumerate(analyses) :
+ self.lcl.append([])
+ self.analyseid = i
+ corpus_uuid = analyse['corpus']
+ #if corpus_uuid not in self.parent.history.openedcorpus :
+ irapath = parametres['corpusira'][i]
+ corpus = Corpus(self, parametres = DoConf(irapath).getoptions('corpus'), read = True)
+ ucepath = os.path.join(analyse['pathout'], 'uce.csv')
+ corpus.make_ucecl_from_R(ucepath)
+ self.ori = corpus
+ for j, cl in enumerate(parametres['clusters'][i]) :
+ #print cl, self.ori.lc[cl-1]
+ self.parametres['uceids'] = self.ori.lc[cl-1]#[st for st in self.ori['lc'][cl-1]]
+ self.lcl[i] += self.ori.lc[cl-1]
+ self.et = parametres['newet'][i][j]
+ self.fromuceids()
+ #create database
+ self.connect()
+ self.dobuild()
+
+ def fromuceids(self):
+ print('fromuceids')
+ dictucekeep = dict(list(zip(self.parametres['uceids'], self.parametres['uceids'])))
+ idpara = 0
+ for uci in self.ori.ucis :
+ if uci.paras == [] :
+ keepuces = [CopyUce(uce) for uce in uci.uces if uce.ident in dictucekeep]
+ if keepuces != [] :
+ nuci = CopyUci(uci)
+ nuci.uces = keepuces
+ nuci.etoiles.append(self.et)
+ nuci.analyseid = self.analyseid
+ self.corpus.ucis.append(nuci)
+ idpara += 1
+ else :
+ newuces = []
+ newpara = []
+ for et in uci.paras :
+ keepuces = [CopyUce(uce) for uce in uci.uces if uce.ident in dictucekeep]
+ idpara += 1
+ if keepuces != [] :
+ newuces += keepuces
+ newpara.append(et)
+ if newuces != [] :
+ nuci = CopyUci(uci)
+ nuci.uces = newuces
+ nuci.paras = newpara
+ nuci.etoiles.append(self.et)
+ nuci.analyseid = self.analyseid
+ self.corpus.ucis.append(nuci)
+ #print nuci.etoiles, nuci.ident, nuci.uces
+
+ def read_corpus(self, infile = None):
+ #self.olduceid = [uce.ident for uci in self.corpus.ucis for uce in uci.uces]
+ ident_uci = 0
+ ident_uce = 0
+ ident_para = -1
+ lastpara = -1
+ newuceident = {}
+ print('redo text, para and st ident')
+ for uci in self.corpus.ucis :
+ #print uci.ident, ident_uci, [uce.ident for uce in uci.uces], uci.etoiles
+ uci.ident = ident_uci
+ ident_uci += 1
+ for uce in uci.uces :
+ uce.uci = uci.ident
+ if uce.para != lastpara :
+ ident_para += 1
+ lastpara = uce.para
+ uce.para = ident_para
+ else :
+ uce.para = ident_para
+ newuceident['%i-%i' %(uci.analyseid, uce.ident)] = ident_uce
+ uce.ident = ident_uce
+ #print uce.ident
+ ident_uce += 1
+ print('backup st text and forms')
+ rowid = 0
+ for i, analyse in enumerate(self.analyses) :
+ #print analyse, self.parametres['corpusira']
+ irapath = self.parametres['corpusira'][i]
+ old = Corpus(self, parametres = DoConf(irapath).getoptions('corpus'), read = True)
+ for row in old.getconcorde(self.lcl[i]) :
+ self.c.execute('INSERT INTO uces VALUES(?,?);', (newuceident['%i-%i' % (i,row[0])], row[1]))
+ for word in row[1].split() :
+ self.corpus.add_word_from_forme(old.formes[word], newuceident['%i-%i' % (i,row[0])])
+ rowid += 1
+ self.backup_uce()
+ print('done')
+
+
+class MergeClusters :
+
+ def __init__(self, parent, parametres = None, dlg = None):
+ self.parent = parent
+ #self.ori = corpus
+ self.dlg = dlg
+ corpus_name = 'MergeFromClusters'
+ if dlg is not None :
+ busy = wx.BusyInfo(_("Please wait..."), self)
+ wx.SafeYield()
+ parametres['corpus_name'] = corpus_name
+ if dlg is not None :
+ del busy
+ dial = MergeClusterFrame(parent)
+ dial.m_textCtrl4.SetValue(corpus_name)
+ self.res = dial.ShowModal()
+ if self.res == 5100 :
+ self.analyses = {}
+ self.clusters = {}
+ self.newet = {}
+ self.corpusira = {}
+ if dial.m_textCtrl4.GetValue() != '' :
+ corpus_name = ''.join([l for l in dial.m_textCtrl4.GetValue() if l.isalnum() or l in ['_']])
+ if corpus_name != '' :
+ parametres['corpus_name'] = corpus_name
+ else :
+ parametres['corpus_name'] = 'MergeFromClusters'
+ for cl in dial.selected :
+ corpus_uuid = cl[1]
+ #if corpus_uuid not in self.parent.history.openedcorpus :
+ irapath = self.parent.history.corpus[corpus_uuid]['ira']
+ #corpus = Corpus(self.parent, parametres = DoConf(irapath).getoptions('corpus'), read = True)
+ #self.parent.history.openedcorpus[corpus_uuid] = corpus
+ if cl[0] not in self.analyses :
+ analyse = DoConf(dial.irapath[cl[0]]).getoptions()
+ #ucepath = os.path.join(os.path.dirname(dial.irapath[cl[0]]), 'uce.csv')
+ #corpus = copycorpus(self.parent.history.openedcorpus[corpus_uuid])
+ #corpus.make_ucecl_from_R(ucepath)
+ self.analyses[cl[0]] = analyse
+ self.clusters[cl[0]] = [cl[2]]
+ self.newet[cl[0]] = [dial.selected[cl]]
+ self.corpusira[cl[0]] = irapath
+ else :
+ self.clusters[cl[0]].append(cl[2])
+ self.newet[cl[0]].append(dial.selected[cl])
+ analyses = [val for val in self.clusters]
+ clusters = [self.clusters[val] for val in analyses]
+ self.newet = [self.newet[val] for val in analyses]
+ corpusira = [self.corpusira[val] for val in analyses]
+ analyses = [self.analyses[val] for val in analyses]
+ pathout = os.path.dirname(os.path.dirname(analyses[0]['pathout']))
+ self.analyses = analyses
+ pathout = os.path.join(pathout, parametres['corpus_name'])
+ i = 1
+ while os.path.exists(pathout + '_%i' % i) :
+ i += 1
+ parametres['pathout'] = pathout + '_%i' % i
+ self.parametres = parametres
+ self.parametres['clusters'] = clusters
+ self.parametres['newet'] = self.newet
+ self.parametres['corpusira'] = corpusira
+ dial.Destroy()
+ else :
+ dial.Destroy()
+
+ def doanalyse(self):
+ return BuildMergeFromClusters(self.analyses, parametres = self.parametres, dlg = self.dlg).corpus
+