X-Git-Url: http://www.iramuteq.org/git?a=blobdiff_plain;f=functions.py;h=3472b77d411a60567df32abdc2f4df7f1aba0854;hb=refs%2Fheads%2F3.0;hp=b41c483545574ab9670bb31f9f4dfb00a1c9284d;hpb=b4ab59a79dbc62d4234778e793c05718648c6775;p=iramuteq diff --git a/functions.py b/functions.py old mode 100644 new mode 100755 index b41c483..4a9b857 --- a/functions.py +++ b/functions.py @@ -1,14 +1,15 @@ -#!/bin/env python # -*- coding: utf-8 -*- #Author: Pierre Ratinaud -#Copyright (c) 2008-2012 Pierre Ratinaud +#Copyright (c) 2008-2020 Pierre Ratinaud +#modification pour python 3 : Laurent Mérat, 6x7 - mai 2020 #License: GNU/GPL -import wx +#------------------------------------ +# import des modules python +#------------------------------------ import re -from ConfigParser import ConfigParser from subprocess import Popen, call, PIPE -import thread +import _thread import os import ast import sys @@ -24,20 +25,32 @@ import shelve import json #from dialog import BugDialog import logging +from operator import itemgetter -log = logging.getLogger('iramuteq') +#------------------------------------ +# import des modules wx +#------------------------------------ +import wx +import wx.adv +#------------------------------------ +# import des fichiers du projet +#------------------------------------ +from configparser import ConfigParser -indices_simi = [u'cooccurrence' ,'pourcentage de cooccurrence',u'Russel',u'Jaccard', 'Kulczynski1', 'Kulczynski2', 'Mountford', 'Fager', 'simple matching', 'Hamman', 'Faith', 'Tanimoto', 'Dice', 'Phi', 'Stiles', 'Michael', 'Mozley', 'Yule', 'Yule2', 'Ochiai', 'Simpson', 'Braun-Blanquet','Chi-squared', 'Phi-squared', 'Tschuprow', 'Cramer', 'Pearson', 'binomial'] + +log = logging.getLogger('iramuteq') +indices_simi = ['cooccurrence' ,'pourcentage de cooccurrence','Russel','Jaccard', 'Kulczynski1', 'Kulczynski2', 'Mountford', 'Fager', 'simple matching', 'Hamman', 'Faith', 'Tanimoto', 'Dice', 'Phi', 'Stiles', 'Michael', 'Mozley', 'Yule', 'Yule2', 'Ochiai', 'Simpson', 'Braun-Blanquet','Chi-squared', 'Phi-squared', 'Tschuprow', 'Cramer', 'Pearson', 'binomial'] def open_folder(folder): if sys.platform == "win32": os.startfile(folder) else: opener ="open" if sys.platform == "darwin" else "xdg-open" - call([opener, folder]) + #call([opener, folder]) + call(["%s '%s' &" % (opener, folder)], shell=True) def normpath_win32(path) : if not sys.platform == 'win32' : @@ -53,10 +66,10 @@ class TGen : self.path = path self.tgen = {} self.encoding = encoding - + def __getitem__(self, key): return self.tgen[key] - + def read(self, path = None): if path is None : path = self.path @@ -66,29 +79,29 @@ class TGen : tgen = dict([[line[0], line[1:]] for line in tgen]) self.tgen = tgen self.path = path - + def write(self, path = None): if path is None : path = self.path - with open(path, 'w') as f : - f.write('\n'.join(['\t'.join([val] + self.tgen[val]) for val in self.tgen]).encode(self.encoding)) - + with open(path, 'w', encoding='utf8') as f : + f.write('\n'.join(['\t'.join([val] + self.tgen[val]) for val in self.tgen])) + def writetable(self, pathout, tgens, totocc): - etoiles = totocc.keys() + etoiles = list(totocc.keys()) etoiles.sort() - with open(pathout, 'w') as f : - line = '\t'.join([u'tgens'] + etoiles) + '\n' - f.write(line.encode(self.encoding)) + with open(pathout, 'w', encoding='utf8') as f : + line = '\t'.join(['tgens'] + etoiles) + '\n' + f.write(line) for t in tgens : - line = '\t'.join([t] + [`tgens[t][et]` for et in etoiles]) + '\n' - f.write(line.encode(self.encoding)) + line = '\t'.join([t] + [repr(tgens[t][et]) for et in etoiles]) + '\n' + f.write(line) i = 0 totname = 'total' - while totname + `i` in tgens : + while totname + repr(i) in tgens : i += 1 - totname = totname + `i` - line = '\t'.join([totname] + [`totocc[et]` for et in etoiles]) + '\n' - f.write(line.encode(self.encoding)) + totname = totname + repr(i) + line = '\t'.join([totname] + [repr(totocc[et]) for et in etoiles]) + '\n' + f.write(line) class History : def __init__(self, filein, syscoding = 'utf8') : @@ -104,7 +117,9 @@ class History : self.read() def read(self) : - d = shelve.open(self.filein) + with open(self.filein, 'r') as fjson : + d = json.load(fjson) +# d = shelve.open(self.filein, protocol=1) self.history = d.get('history', []) self.matrix = d.get('matrix', []) self.ordercorpus = dict([[corpus['uuid'], i] for i, corpus in enumerate(self.history)]) @@ -112,13 +127,16 @@ class History : self.analyses = dict([[analyse['uuid'], analyse] for corpus in self.history for analyse in corpus.get('analyses', [])]) self.matrixanalyse = dict([[mat['uuid'], mat] for mat in self.matrix]) self.ordermatrix = dict([[matrix['uuid'], i] for i, matrix in enumerate(self.matrix)]) - d.close() +# d.close() def write(self) : - d = shelve.open(self.filein) + d = {} d['history'] = self.history d['matrix'] = self.matrix - d.close() + with open(self.filein, 'w') as f : + f.write(json.dumps(d, indent=4, default=str)) + #d = shelve.open(self.filein, protocol=1) + #d.close() def add(self, analyse) : log.info('add to history %s' % analyse.get('corpus_name', 'pas un corpus')) @@ -190,6 +208,11 @@ class History : self.history[self.ordercorpus[analyse['corpus']]]['analyses'].pop(todel) elif analyse['uuid'] in self.matrixanalyse : self.matrix = [mat for mat in self.matrix if mat['uuid'] != analyse['uuid']] + elif analyse.get('matrix', False) in self.matrixanalyse : + analyses = self.matrix[self.ordermatrix[analyse['matrix']]]['analyses'] + topop = [i for i, val in enumerate(analyses) if analyse['uuid'] == val['uuid']][0] + analyses.pop(topop) + self.matrix[self.ordermatrix[analyse['matrix']]]['analyses'] = analyses self.write() self.read() @@ -214,26 +237,83 @@ class History : def clean(self) : corpustodel = [corpus for corpus in self.history if not os.path.exists(corpus['ira'])] - print corpustodel + print(corpustodel) for corpus in corpustodel : - print 'cleaning :', corpus['corpus_name'] + print('cleaning :', corpus['corpus_name']) self.delete(corpus, corpus = True) anatodel = [analyse for corpus in self.history for analyse in corpus.get('analyses', []) if not os.path.exists(analyse.get('ira', '/'))] for analyse in anatodel : - print 'cleaning :', analyse['name'] + print('cleaning :', analyse['name']) self.delete(analyse) + def dostat(self): + todel = {} + tokens = 0 + corpusnb = {} + subnb = 0 + analysenb = 0 + hours = 0 + minutes = 0 + secondes = 0 + ha = 0 + ma = 0 + sa = 0 + for corpus in self.history : + analysenb += len(corpus.get('analyses', [])) + analyses = corpus.get('analyses', []) + for analyse in analyses : + if os.path.exists(analyse['ira']) : + ana = DoConf(analyse['ira']).getoptions() + if 'time' in ana : + time = ana['time'].split() + ha += int(time[0].replace('h','')) * 3600 + ma += int(time[1].replace('m','')) * 60 + sa += int(time[2].replace('s','')) + if os.path.exists(corpus['ira']) : + param = DoConf(corpus['ira']).getoptions() + time = param.get('time','0h 0m 0s') + time = time.split() + hours += int(time[0].replace('h','')) * 3600 + minutes += int(time[1].replace('m','')) * 60 + secondes += int(time[2].replace('s','')) + if param.get('originalpath', False) : + if param['originalpath'] in corpusnb : + corpusnb[param['originalpath']] += 1 + tokens += int(param['occurrences']) + else : + corpusnb[param['originalpath']] = 1 + #print param + else : + subnb += 1 + else : + if corpus['ira'] in todel : + todel['ira'] += 1 + else : + todel['ira'] = 1 + print('Nbr total de corpus : %s' % len(self.history)) + corpus_nb = len(corpusnb) + len(todel) + print('Nbr de corpus différents : %s' % corpus_nb) + lentodel = len(todel) + print('Nbr de corpus à supprimer : %s' % lentodel) + print('Nbr de sous corpus : %s' % subnb) + print("Nbr total d'occurrences : %s" % tokens) + print('Moyenne occurrences par corpus : %f' % (tokens/corpus_nb)) + print('---------------------') + print("Nbr total d'analyses : %s" % analysenb) + print('Temps total indexation : %f h' % ((hours+minutes+secondes) / 3600)) + print('Temps total analyses : %f h' % ((ha+ma+sa) / 3600)) + def __str__(self) : return str(self.history) class DoConf : def __init__(self, configfile=None, diff = None, parametres = None) : self.configfile = configfile - self.conf = ConfigParser() + self.conf = ConfigParser(interpolation=None) # pourquoi ce paramètre ??? if configfile is not None : configfile = normpath_win32(configfile) - self.conf.readfp(codecs.open(configfile, 'r', 'utf8')) + self.conf.read_file(codecs.open(configfile, 'r', 'utf8')) self.parametres = {} if parametres is not None : self.doparametres(parametres) @@ -273,10 +353,10 @@ class DoConf : self.conf.add_section(section) for option in parametres[i] : if isinstance(parametres[i][option], int) : - self.conf.set(section, option, `parametres[i][option]`) + self.conf.set(section, option, repr(parametres[i][option])) txt += '%s = %i\n' % (option, parametres[i][option]) - elif isinstance(parametres[i][option], basestring) : - self.conf.set(section, option, parametres[i][option].encode('utf8')) + elif isinstance(parametres[i][option], str) : + self.conf.set(section, option, parametres[i][option]) txt += '%s = %s\n' % (option, parametres[i][option]) elif isinstance(parametres[i][option], wx.Colour) : self.conf.set(section, option, str(parametres[i][option])) @@ -284,13 +364,13 @@ class DoConf : elif option == 'analyses' : pass else : - self.conf.set(section, option, `parametres[i][option]`) - txt += '%s = %s\n' % (option, `parametres[i][option]`) + self.conf.set(section, option, repr(parametres[i][option])) + txt += '%s = %s\n' % (option, repr(parametres[i][option])) if outfile is None : outfile = self.configfile outfile = normpath_win32(outfile) - with open(outfile, 'w') as f : - f.write(txt.encode('utf8')) + with open(outfile, 'w', encoding="utf-8") as f : + f.write(txt) #self.conf.write(f) def totext(self, parametres) : @@ -298,19 +378,19 @@ class DoConf : txt = [] for val in parametres : if isinstance(parametres[val], int) : - txt.append(' \t\t: '.join([val, `parametres[val]`])) - elif isinstance(parametres[val], basestring) : + txt.append(' \t\t: '.join([val, repr(parametres[val])])) + elif isinstance(parametres[val], str) : txt.append(' \t\t: '.join([val, parametres[val]])) elif val in ['listet', 'stars'] : pass else : - txt.append(' \t\t: '.join([val, `parametres[val]`])) + txt.append(' \t\t: '.join([val, repr(parametres[val])])) return '\n'.join(txt) def write_tab(tab, fileout) : - writer = csv.writer(open(fileout, 'wb'), delimiter=';', quoting = csv.QUOTE_NONNUMERIC) - writer.writerows(tab) + csvWriter = csv.writer(open(fileout, 'w', newline='', encoding='utf8'), delimiter=';', quoting = csv.QUOTE_NONNUMERIC) + csvWriter.writerows(tab) class BugDialog(wx.Dialog): def __init__(self, *args, **kwds): @@ -353,11 +433,28 @@ def CreateIraFile(DictPathOut, clusternb, corpname='corpus_name', section = 'ana AnalyseConf.set(section, 'clusternb', clusternb) AnalyseConf.set(section, 'corpus_name', corpname) - fileout = open(DictPathOut['ira'], 'w') + fileout = open(DictPathOut['ira'], 'w', encoding='utf8') AnalyseConf.write(fileout) fileout.close() -def sortedby(list, direct, *indices): +def multisort(liste2d, ordre, indices_tri): + + """ + methode destinée à remplacer 'comp' qui a disparu en Python 3 + tri de tuples sur l'un des éléments du tuple + en principe, elle doit renvoyer les éléments triés selon le principe d'avant + tel que décrit dans la docstring de 'sortedby' + + probablement à améliorer pour la rendre d'usage plus général + en acceptant un nombre variable de parametres ??? + """ + + indices_triTuple = indices_tri.Tuple(int, ...) + for key in reversed(indices_tri): + liste2d.sort(key=attrgetter(key), reverse=ordre) + return liste2d + +def sortedby(liste2d, direct, *indices): """ sortedby: sort a list of lists (e.g. a table) by one or more indices @@ -367,16 +464,36 @@ def sortedby(list, direct, *indices): for list = [[2,3],[1,2],[3,1]]: sortedby(list,1) will return [[3, 1], [1, 2], [2, 3]], sortedby(list,0) will return [[1, 2], [2, 3], [3, 1]] + + elle n'est pas remplacée par la méthode 'multisort' ??? + """ - nlist = map(lambda x, indices=indices: - map(lambda i, x=x: x[i], indices) + [x], - list) - if direct == 1: - nlist.sort() - elif direct == 2: - nlist.sort(reverse=True) - return map(lambda l: l[-1], nlist) +# iramuteq original +# nlist = map(lambda x, indices=indices: +# map(lambda i, x=x: x[i], indices) + [x], +# list) + +# iramuteq passé à 2to3 +# nlist = list(map(lambda x, indices=indices: +# list(map(lambda i, x=x: x[i], indices)) + [x], +# liste2d)) + + for key in reversed(indices): + liste2d.sort(key=itemgetter(key), reverse=(direct==2)) + return liste2d + + +# if direct == 1: +# nlist.sort() +# sorted_list = multisort(liste2d, direct, *indices) + +# elif direct == 2: +# nlist.sort(reverse=True) +# sorted_list = multisort(liste2d, direct, *indices) + +# return [l[-1] for l in nlist] +# return sorted_list def add_type(line, dictlem): if line[4] in dictlem: @@ -389,7 +506,7 @@ def treat_line_alceste(i, line) : if line[0] == '*' or line[0] == '*****' : return line + [''] if line[5] == 'NA': - print 'NA', line[5] + print('NA', line[5]) pass elif float(line[5].replace(',', '.')) < 0.0001: line[5] = '< 0,0001' @@ -399,10 +516,10 @@ def treat_line_alceste(i, line) : line[5] = str(float(line[5].replace(',', '.')))[0:7] return [i, int(line[0]), int(line[1]), float(line[2]), float(line[3]), line[6], line[4], line[5]] -def ReadProfileAsDico(File, Alceste=False, encoding = sys.getdefaultencoding()): +def ReadProfileAsDico(File, Alceste=False, encoding = 'utf8'): dictlem = {} - print 'lecture des profiles' - FileReader = codecs.open(File, 'r', encoding) + print('lecture des profiles') + FileReader = open(File, 'r', encoding='utf8') Filecontent = FileReader.readlines() FileReader.close() DictProfile = {} @@ -412,13 +529,13 @@ def ReadProfileAsDico(File, Alceste=False, encoding = sys.getdefaultencoding()): rows.pop(0) ClusterNb = rows[0][2] rows.pop(0) - clusters = [row[2] for row in rows if row[0] == u'**'] - valclusters = [row[1:4] for row in rows if row[0] == u'****'] - lp = [i for i, line in enumerate(rows) if line[0] == u'****'] - prof = [rows[lp[i] + 1:lp[i+1] - 1] for i in range(0, len(lp)-1)] + [rows[lp[-1] + 1:len(rows)]] + clusters = [row[2] for row in rows if row[0] == '**'] + valclusters = [row[1:4] for row in rows if row[0] == '****'] + lp = [i for i, line in enumerate(rows) if line[0] == '****'] + prof = [rows[lp[i] + 1:lp[i+1] - 1] for i in range(0, len(lp)-1)] + [rows[lp[-1] + 1:len(rows)]] if Alceste : prof = [[add_type(row, dictlem) for row in pr] for pr in prof] - prof = [[treat_line_alceste(i,line) for i, line in enumerate(pr)] for pr in prof] + prof = [[treat_line_alceste(i,line) for i, line in enumerate(pr)] for pr in prof] else : prof = [[line + [''] for line in pr] for pr in prof] prof = [[treat_line_alceste(i,line) for i, line in enumerate(pr)] for pr in prof] @@ -455,18 +572,18 @@ def decoupercharact(chaine, longueur, longueurOptimale, separateurs = None) : Si on trouve un '$', c'est fini. Sinon, on cherche le meilleur candidat. C'est-à-dire le rapport poids/distance le plus important. """ - separateurs = [[u'.', 60.0], [u'?', 60.0], [u'!', 60.0], [u'£$£', 60], [u':', 50.0], [u';', 40.0], [u',', 10.0], [u' ', 0.1]] + separateurs = [['.', 60.0], ['?', 60.0], ['!', 60.0], ['£$£', 60], [':', 50.0], [';', 40.0], [',', 10.0], [' ', 0.1]] trouve = False # si on a trouvé un bon séparateur iDecoupe = 0 # indice du caractere ou il faut decouper - + # on découpe la chaine pour avoir au maximum 240 caractères longueur = min(longueur, len(chaine) - 1) chaineTravail = chaine[:longueur + 1] nbCar = longueur meilleur = ['', 0, 0] # type, poids et position du meilleur separateur - + # on vérifie si on ne trouve pas un '$' - indice = chaineTravail.find(u'$') + indice = chaineTravail.find('$') if indice > -1: trouve = True iDecoupe = indice @@ -481,8 +598,8 @@ def decoupercharact(chaine, longueur, longueurOptimale, separateurs = None) : # on vérifie si le caractére courant est une marque de ponctuation for s in separateurs: if caractere == s[0]: - # si c'est une ponctuation - + # si c'est une ponctuation + if s[1] / distance > float(meilleur[1]) / meilleureDistance: # print nbCar, s[0] meilleur[0] = s[0] @@ -490,13 +607,13 @@ def decoupercharact(chaine, longueur, longueurOptimale, separateurs = None) : meilleur[2] = nbCar trouve = True iDecoupe = nbCar - + # et on termine la recherche break # on passe au caractère précédant nbCar = nbCar - 1 - + # si on a trouvé if trouve: fin = chaine[iDecoupe + 1:] @@ -506,20 +623,20 @@ def decoupercharact(chaine, longueur, longueurOptimale, separateurs = None) : return False, chaine.split(), '' -exceptions = {'paragrapheOT' : u"Un problème de formatage (présence d'un marqueur de paragraphe (-*) en dehors d'un texte) est survenu à la ligne ", - 'EmptyText' : u"Texte vide (probablement un problème de formatage du corpus). Le problème est apparu à la ligne ", - 'CorpusEncoding' : u"Problème d'encodage.", - 'TextBeforeTextMark' : u"Problème de formatage : du texte avant le premier marqueur de texte (****). Le problème est survenu à la ligne ", - 'MissingAnalyse' : u'Aucun fichier à cet emplacement :\n', +exceptions = {'paragrapheOT' : "Un problème de formatage (présence d'un marqueur de paragraphe (-*) en dehors d'un texte) est survenu à la ligne ", + 'EmptyText' : "Texte vide (probablement un problème de formatage du corpus). Le problème est apparu à la ligne ", + 'CorpusEncoding' : "Problème d'encodage.", + 'TextBeforeTextMark' : "Problème de formatage : du texte avant le premier marqueur de texte (****). Le problème est survenu à la ligne ", + 'MissingAnalyse' : 'Aucun fichier à cet emplacement :\n', } def BugReport(parent, error = None): for ch in parent.GetChildren(): if "" == str(type(ch)): - ch.Destroy() + ch.Destroy() excName, exc, excTb = formatExceptionInfo() if excName == 'Exception' : - print exc + print(exc) if len(exc.split()) == 2 : mss, linenb = exc.split() if mss in exceptions : @@ -533,11 +650,11 @@ def BugReport(parent, error = None): txt = exc title = "Information" else : - txt = u' !== BUG ==! \n' - txt += u'*************************************\n' + txt = '\n !== BUG ==! \n' + txt += '*************************************\n' txt += '\n'.join(excTb).replace(' ', ' ') txt += excName + '\n' - txt += `exc` + txt += repr(exc) title = "Bug" dial = BugDialog(parent, **{'title' : title}) @@ -549,22 +666,22 @@ def BugReport(parent, error = None): dial.CenterOnParent() dial.ShowModal() dial.Destroy() - + def PlaySound(parent): if parent.pref.getboolean('iramuteq', 'sound') : try: if "gtk2" in wx.PlatformInfo: error = Popen(['aplay','-q',os.path.join(parent.AppliPath,'son_fin.wav')]) - else : - sound = wx.Sound(os.path.join(parent.AppliPath, 'son_fin.wav')) - sound.Play(wx.SOUND_SYNC) + else : + sound = wx.adv.Sound(os.path.join(parent.AppliPath, 'son_fin.wav')) + sound.Play(wx.adv.SOUND_SYNC) except : - print 'pas de son' + print('pas de son') def ReadDicoAsDico(dicopath): - with codecs.open(dicopath, 'r', 'UTF8') as f: + with open(dicopath, 'r', encoding='UTF8') as f: content = f.readlines() - lines = [line.rstrip('\n\r').replace(u'\n', '').replace('"', '').split('\t') for line in content if line != u''] + lines = [line.rstrip('\n\r').replace('\n', '').replace('"', '').split('\t') for line in content if line != ''] return dict([[line[0], line[1:]] for line in lines]) def ReadLexique(parent, lang = 'french', filein = None): @@ -579,9 +696,9 @@ def ReadLexique(parent, lang = 'french', filein = None): else : parent.lexique = ReadDicoAsDico(filein) -def ReadList(filein, encoding = sys.getdefaultencoding(), sep = ';'): +def ReadList(filein, encoding = 'utf8', sep = ';'): #file = open(filein) - with codecs.open(filein, 'r', encoding) as f : + with open(filein, 'r', encoding='utf8') as f : content = f.read() content = [line.replace('\n', '').replace('\r','').replace('\"', '').replace(',', '.').split(sep) for line in content.splitlines()] #file = codecs.open(filein, 'r', encoding) @@ -596,9 +713,9 @@ def ReadList(filein, encoding = sys.getdefaultencoding(), sep = ';'): #line = line.split(';') nline = [line[0]] for val in line[1:]: - if val == u'NA' : + if val == 'NA' : don = '' - else: + else: try: don = int(val) except: @@ -626,7 +743,6 @@ def exec_rcode(rpath, rcode, wait = True, graph = False): needX11 = False except : needX11 = False - rpath = rpath.replace('\\','\\\\') env = os.environ.copy() if sys.platform == 'darwin' and 'LC_ALL' not in env: @@ -636,13 +752,13 @@ def exec_rcode(rpath, rcode, wait = True, graph = False): if sys.platform == 'win32': error = call(["%s" % rpath, "--vanilla","--slave","-f", "%s" % rcode]) else : - error = call([rpath, '--slave', "--vanilla", "-f %s" % rcode, "--encoding=UTF-8"], env = env) + error = call([rpath, '--slave', "--vanilla", "--encoding=UTF-8", "-f %s" % rcode], env = env) return error else : if sys.platform == 'win32': pid = Popen(["%s" % rpath, '--vanilla','--slave','-f', "%s" % rcode]) else : - pid = Popen([rpath, '--slave', "--vanilla", "-f %s" % rcode, "--encoding=UTF-8"], stderr = PIPE, env = env) + pid = Popen([rpath, '--slave', "--vanilla", "--encoding=UTF-8", "-f %s" % rcode], stderr = PIPE, env = env, encoding='UTF-8') #PIPE ou STDOUT ? return pid else : if wait : @@ -650,18 +766,18 @@ def exec_rcode(rpath, rcode, wait = True, graph = False): error = call(["%s" % rpath, '--vanilla','--slave','-f', "%s" % rcode]) elif sys.platform == 'darwin' and needX11: os.environ['DISPLAY'] = ':0.0' - error = call([rpath, '--vanilla','--slave',"-f %s" % rcode, "--encoding=UTF-8"], env = env) + error = call([rpath, '--vanilla','--slave', "--encoding=UTF-8","-f %s" % rcode], env = env, encoding='UTF-8') else : - error = call([rpath, '--vanilla','--slave',"-f %s" % rcode, "--encoding=UTF-8"], env = env) + error = call([rpath, '--vanilla','--slave', "--encoding=UTF-8","-f %s" % rcode], env = env, encoding='UTF-8') return error else : if sys.platform == 'win32': pid = Popen(["%s" % rpath, '--vanilla','--slave','-f', "%s" % rcode]) elif sys.platform == 'darwin' and needX11: os.environ['DISPLAY'] = ':0.0' - pid = Popen([rpath, '--vanilla','--slave',"-f %s" % rcode, "--encoding=UTF-8"], stderr = PIPE, env = env) + pid = Popen([rpath, '--vanilla','--slave', "--encoding=UTF-8","-f %s" % rcode], stderr = PIPE, env = env, encoding='UTF-8') else : - pid = Popen([rpath, '--vanilla','--slave',"-f %s" % rcode, "--encoding=UTF-8"], stderr = PIPE, env = env) + pid = Popen([rpath, '--vanilla','--slave', "--encoding=UTF-8","-f %s" % rcode], stderr = PIPE, env = env, encoding='UTF-8') return pid def check_Rresult(parent, pid) : @@ -673,7 +789,7 @@ def check_Rresult(parent, pid) : error[1] = 'None' parent.Rerror = '\n'.join([str(pid.returncode), '\n'.join(error)]) try : - raise Exception('\n'.join([u'Erreur R', '\n'.join(error[1:])])) + raise Exception('\n'.join(['Erreur R', '\n'.join(error[1:])])) except : BugReport(parent) return False @@ -682,7 +798,7 @@ def check_Rresult(parent, pid) : else : if pid != 0 : try : - raise Exception(u'Erreur R') + raise Exception('Erreur R') except : BugReport(parent) return False @@ -694,22 +810,22 @@ def launchcommand(mycommand): Popen(mycommand) def print_liste(filename,liste): - with open(filename,'w') as f : + with open(filename,'w', encoding='utf8') as f : for graph in liste : - f.write(';'.join(graph).encode(sys.getdefaultencoding())+'\n') + f.write(';'.join(graph) +'\n') -def read_list_file(filename, encoding = sys.getdefaultencoding()): - with codecs.open(filename,'rU', encoding) as f : +def read_list_file(filename, encoding = 'utf8'): + with open(filename,'r', encoding='utf8') as f: content=f.readlines() ncontent=[line.replace('\n','').split(';') for line in content if line.strip() != ''] return ncontent -def progressbar(self, maxi) : +def progressbar(self, maxi): ira = wx.GetApp().GetTopWindow() parent = ira - try : + try: maxi = int(maxi) - except : + except: maxi = 1 prog = wx.ProgressDialog("Traitements", "Veuillez patienter...", @@ -717,6 +833,8 @@ def progressbar(self, maxi) : parent=parent, style=wx.PD_APP_MODAL | wx.PD_AUTO_HIDE | wx.PD_ELAPSED_TIME | wx.PD_CAN_ABORT ) + # parent ??? + # le ABORT n'est pas géré à tous les coups ??? prog.SetSize((400,150)) #prog.SetIcon(ira._icon) return prog @@ -731,8 +849,8 @@ def treat_var_mod(variables) : var_mod[var] = mods # for variable in variables : -# if u'_' in variable : -# forme = variable.split(u'_') +# if '_' in variable : +# forme = variable.split('_') # var = forme[0] # mod = forme[1] # if not var in var_mod : @@ -753,12 +871,13 @@ def doconcorde(corpus, uces, mots, uci = False) : listmot = [corpus.getlems()[lem].formes for lem in mots] listmot = [corpus.getforme(fid).forme for lem in listmot for fid in lem] mothtml = ['%s' % mot for mot in listmot] - dmots = dict(zip(listmot, mothtml)) + dmots = dict(list(zip(listmot, mothtml))) for uce in uces : ucetxt = ucestxt1[uce].split() ucetxt = ' '.join([dmots.get(mot, mot) for mot in ucetxt]) if not uci : - ucis_txt.append('

' + ' '.join(corpus.ucis[corpus.getucefromid(uce).uci].etoiles) + '

') + uciid = corpus.getucefromid(uce).uci + ucis_txt.append('

' + ' '.join(corpus.ucis[corpus.getucefromid(uce).uci].etoiles) + ' *%i_%i

' % (uciid, uce, uciid, uce)) else : ucis_txt.append('

' + ' '.join(corpus.ucis[uce].etoiles) + '

') ucestxt.append(ucetxt) @@ -767,8 +886,8 @@ def doconcorde(corpus, uces, mots, uci = False) : def getallstcarac(corpus, analyse) : pathout = PathOut(analyse['ira']) - profils = ReadProfileAsDico(pathout['PROFILE_OUT'], Alceste, self.encoding) - print profils + profils = ReadProfileAsDico(pathout['PROFILE_OUT'], Alceste, 'utf8') + print(profils) def read_chd(filein, fileout): with open(filein, 'r') as f : @@ -800,7 +919,7 @@ translation_languages = {"Afrikaans":"af", "Albanian":"sq", "Amharic":"am", "Ara def gettranslation(words, lf, lt) : - import urllib2 + import urllib.request, urllib.error, urllib.parse import json agent = {'User-Agent': "Mozilla/4.0 (\ @@ -813,13 +932,13 @@ def gettranslation(words, lf, lt) : .NET CLR 3.0.04506.30\ )"} base_link = "https://translate.googleapis.com/translate_a/single?client=gtx&sl=%s&tl=%s&dt=t&q=%s" - print len(words) - totrans = urllib2.quote('\n'.join(words).encode('utf8')) + print(len(words)) + totrans = urllib.parse.quote('\n'.join(words)) link = base_link % (lf, lt, totrans) - request = urllib2.Request(link, headers=agent) - raw_data = urllib2.urlopen(request).read() + request = urllib.request.Request(link, headers=agent) + raw_data = urllib.request.urlopen(request).read() data = json.loads(raw_data) - return [line[0].decode('utf8').replace(u"'", u'_').replace(u' | ', u'|').replace(u' ', u'_').replace(u'-',u'_').replace(u'\n','') for line in data[0]] + return [line[0].replace("'", '_').replace(' | ', '|').replace(' ', '_').replace('-','_').replace('\n','') for line in data[0]] def makenprof(prof, trans, deb=0) : nprof=[] @@ -837,50 +956,50 @@ def treatempty(val) : else : return val -def translateprofile(corpus, dictprofile, lf='it', lt='fr') : +def translateprofile(corpus, dictprofile, lf='it', lt='fr', maxword = 50) : nprof = {} lems = {} for i in range(len(dictprofile)) : - prof = dictprofile[`i+1`] + prof = dictprofile[repr(i+1)] try : - lenact = prof.index([u'*****', u'*', u'*', u'*', u'*', u'*', '', '']) + lenact = prof.index(['*****', '*', '*', '*', '*', '*', '', '']) lensup = -1 except ValueError: try : - lenact = prof.index([u'*', u'*', u'*', u'*', u'*', u'*', '', '']) + lenact = prof.index(['*', '*', '*', '*', '*', '*', '', '']) lensup = 0 except ValueError: lenact = len(prof) lensup = 0 try : - lensup += prof.index([u'*', u'*', u'*', u'*', u'*', u'*', '', '']) + lensup += prof.index(['*', '*', '*', '*', '*', '*', '', '']) lensup = lensup - lenact except ValueError: lensup += len(prof) - lenact if lenact != 0 : - if lenact > 400 : - nlenact = 400 + if lenact > maxword : + nlenact = maxword else : nlenact = lenact actori = [line[6] for line in prof[1:nlenact]] - act = [val.replace(u'_', u' ') for val in actori] + act = [val.replace('_', ' ') for val in actori] act = gettranslation(act, lf, lt) for j, val in enumerate(actori) : if act[j] not in lems : lems[act[j]] = val else : while act[j] in lems : - act[j] = act[j] + u"+" + act[j] = act[j] + "+" lems[act[j]] = val - nprof[`i+1`] = makenprof(prof, act) + nprof[repr(i+1)] = makenprof(prof, act) if lensup != 0 : - if lensup > 400 : - nlensup = 400 + if lensup > maxword : + nlensup = maxword else : nlensup = lensup supori = [line[6] for line in prof[(1+lenact):(lenact+nlensup)]] - sup = [val.replace(u'_', u' ') for val in supori] + sup = [val.replace('_', ' ') for val in supori] sup = [treatempty(val) for val in sup] sup = gettranslation(sup, lf, lt) for j, val in enumerate(supori) : @@ -888,45 +1007,87 @@ def translateprofile(corpus, dictprofile, lf='it', lt='fr') : lems[sup[j]] = val else : while sup[j] in lems : - sup[j] = sup[j] + u"+" + sup[j] = sup[j] + "+" lems[sup[j]] = val - nprof[`i+1`].append([u'*****', u'*', u'*', u'*', u'*', u'*', '', '']) - nprof[`i+1`] += makenprof(prof, sup, deb=lenact) + nprof[repr(i+1)].append(['*****', '*', '*', '*', '*', '*', '', '']) + nprof[repr(i+1)] += makenprof(prof, sup, deb=lenact) try : - lenet = prof.index([u'*', u'*', u'*', u'*', u'*', u'*', '', '']) - nprof[`i+1`].append([u'*', u'*', u'*', u'*', u'*', u'*', '', '']) - nprof[`i+1`] += prof[(lenet+1):] + lenet = prof.index(['*', '*', '*', '*', '*', '*', '', '']) + nprof[repr(i+1)].append(['*', '*', '*', '*', '*', '*', '', '']) + nprof[repr(i+1)] += prof[(lenet+1):] except : pass return nprof, lems - def write_translation_profile(prof, lems, language, dictpathout) : if os.path.exists(dictpathout['translations.txt']) : - with codecs.open(dictpathout['translations.txt'], 'r', 'utf8') as f : + with open(dictpathout['translations.txt'], 'r', encoding='utf8') as f : translist = f.read() translist = [line.split('\t') for line in translist.splitlines()] else : translist = [] toprint = [] toprint.append(['','','','','','']) - toprint.append([u'***', u'nb classes', `len(prof)`, u'***', '', '']) + toprint.append(['***', 'nb classes', repr(len(prof)), '***', '', '']) for i in range(len(prof)) : - toprint.append([u'**', u'classe', `i+1`, u'**', '', '']) - toprint.append([u'****'] + prof[`i+1`][0] + [u'****']) - rest = [[`line[1]`, `line[2]`, `line[3]`, `line[4]`, line[6], line[7].replace('< 0,0001', '0.00009').replace('NS (','').replace(')','')] for line in prof[`i+1`][1:]] - for i, line in enumerate(prof[`i+1`][1:]) : - if line[0] == u'*' : - rest[i] = [u'*', u'*', u'*', u'*', u'*', u'*'] - elif line[0] == u'*****' : - rest[i] = [u'*****',u'*',u'*', u'*', u'*', u'*'] + toprint.append(['**', 'classe', repr(i+1), '**', '', '']) + toprint.append(['****'] + prof[repr(i+1)][0] + ['****']) + rest = [[repr(line[1]), repr(line[2]), repr(line[3]), repr(line[4]), line[6], line[7].replace('< 0,0001', '0.00009').replace('NS (','').replace(')','')] for line in prof[repr(i+1)][1:]] + for i, line in enumerate(prof[repr(i+1)][1:]) : + if line[0] == '*' : + rest[i] = ['*', '*', '*', '*', '*', '*'] + elif line[0] == '*****' : + rest[i] = ['*****','*','*', '*', '*', '*'] toprint += rest - with open(dictpathout['translation_profile_%s.csv' % language], 'w') as f : - f.write('\n'.join([';'.join(line) for line in toprint]).encode('utf8')) - with open(dictpathout['translation_words_%s.csv' % language], 'w') as f : - f.write('\n'.join(['\t'.join([val, lems[val]]) for val in lems]).encode('utf8')) + with open(dictpathout['translation_profile_%s.csv' % language], 'w', encoding='utf8') as f : + f.write('\n'.join([';'.join(line) for line in toprint])) + with open(dictpathout['translation_words_%s.csv' % language], 'w', encoding='utf8') as f : + f.write('\n'.join(['\t'.join([val, lems[val]]) for val in lems])) if 'translation_profile_%s.csv' % language not in [val[0] for val in translist] : translist.append(['translation_profile_%s.csv' % language, 'translation_words_%s.csv' % language]) - with open(dictpathout['translations.txt'], 'w') as f : - f.write('\n'.join(['\t'.join(line) for line in translist]).encode('utf8')) + with open(dictpathout['translations.txt'], 'w', encoding='utf8') as f : + f.write('\n'.join(['\t'.join(line) for line in translist])) + +def makesentidict(infile, language) : + with codecs.open(infile,'r', 'utf8') as f : + content = f.read() + content = [line.split('\t') for line in content.splitlines()] + titles = content.pop(0) + senti = ['Positive', 'Negative', 'Anger', 'Anticipation', 'Disgust', 'Fear', 'Joy', 'Sadness', 'Surprise', 'Trust'] + sentid = {} + for sent in senti : + sentid[sent] = titles.index(sent) + frtitle = [val for val in titles if '(fr)' in val] + frid = titles.index(frtitle[0]) + sentidict = [[line[frid].lower(), [line[sentid[sent]] for sent in senti]] for line in content] + pos = ['positive'] + [line[0] for line in sentidict if line[1][0] == '1'] + neg = ['negative'] + [line[0] for line in sentidict if line[1][1] == '1'] + anger = ['anger'] + [line[0] for line in sentidict if line[1][2] == '1'] + anticipation = ['anticipation'] + [line[0] for line in sentidict if line[1][3] == '1'] + disgust = ['disgust'] + [line[0] for line in sentidict if line[1][4] == '1'] + fear = ['fear'] + [line[0] for line in sentidict if line[1][5] == '1'] + joy = ['joy'] + [line[0] for line in sentidict if line[1][6] == '1'] + sadness = ['sadness'] + [line[0] for line in sentidict if line[1][7] == '1'] + surprise = ['surprise'] + [line[0] for line in sentidict if line[1][8] == '1'] + trust = ['trust'] + [line[0] for line in sentidict if line[1][9] == '1'] + with open('/tmp/tgenemo.csv', 'w') as f : + for val in [pos, neg, anger, anticipation, disgust, fear, joy, sadness, surprise, trust] : + f.write('\t'.join(val) + '\n') + +def countsentfromprof(prof, encoding, sentidict) : + with codecs.open(prof, 'r', encoding) as f : + content = f.read() + content = [line.split(';') for line in content.splitlines()] + print(content) + content = [[line[0], [int(val) for val in line[1:]]] for line in content] + print(content) + content = dict(content) + print(content) + +def iratolexico(infile, outfile, encoding) : + with codecs.open(infile, 'r', encoding) as f : + for line in f : + if line.startswith('**** ') : + line = line.split() +