2 # -*- coding: utf-8 -*-
3 #Author: Pierre Ratinaud
5 #a simple treetagger parser
9 from functions import ReadDicoAsDico
10 from itertools import izip, chain
13 #infile = '/home/pierre/prog/treetagger/cmd/allenglish.txt'
15 infile = '/home/pierre/fac/cablegate/cablestagger.txt'
16 word_list = '/home/pierre/fac/cablegate/liste_de_mots.csv'
17 actives_list = '/home/pierre/fac/cablegate/actives.csv'
18 supps_list = '/home/pierre/fac/cablegate/supplementaires.csv'
21 def prepare_for_treetagger(corpus, parent) :
22 fileout = '/home/pierre/workspace/iramuteq/corpus/corpus_pour_tt.txt'
24 lang = corpus.parametre['lang']
25 dico_path = parent.DictPath.get(lang + '_exp', 'french_exp')
26 expressions = ReadDicoAsDico(dico_path)
27 corpus.find_expression(expressions)
28 #corpus.content = re.sub(u'[-]+', ' ', corpus.content)
29 corpus.content = re.sub(u'[ ]+', ' ', corpus.content)
30 #FIXME : remplacer . par ' . '
31 #corpus.quick_clean2()
32 with open(fileout, 'w') as f :
33 f.write(corpus.content)
35 def partition(alist, indices):
36 pairs = izip(chain([0], indices), chain(indices, [None]))
37 return (alist[i:j] for i, j in pairs)
39 def partition_uci(uci) :
43 indices = [i for i, forme in enumerate(uci) if forme[0].startswith(u'*')]
44 pairs = izip(chain([0], indices), chain(indices, [None]))
45 return (uci[i,j] for i, j in pairs)
47 def dodict(inlist, indict) :
48 for i, forme in enumerate(inlist) :
49 if tuple(forme) in tot :
50 tot[tuple(forme)].append(i)
52 tot[tuple(forme)] = [i]
54 def treat_forme(forme, tformes) :
55 ponct =[u',', u'', u'``', u"''", u':', u'#', u')', '(', u'!', u'?', u';', u'-', u'.', u'...']
56 if forme[0] in ponct :
58 tforme = (forme[0], forme[1].split(':')[0])
59 return tformes[tforme]
61 def make_formes_and_lems(inlist) :
68 gram = forme[1].split(':')[0]
71 if tforme in tformes :
72 if formes_lems[tformes[tforme]] == u'<unknown>' and lem != u'<unknown>' :
73 formes_lems[tformes[tforme]] = lem
76 nword = u'@'.join([word,''])
77 while nword in formes and formes[nword] != gram :
78 nword = u'@'.join([nword,''])
79 formes[nword] = [0, {}, gram, len(formes)]
80 tformes[tforme] = nword
81 formes_lems[nword] = lem
83 formes[word] = [0, {}, gram, len(formes)]
84 tformes[tforme] = word
85 formes_lems[word] = lem
87 if formes_lems[forme] == u'<unknown>' :
88 formes_lems[forme] = forme
89 if formes_lems[forme] in lems :
90 lems[formes_lems[forme]].append(forme)
92 lems[formes_lems[forme]] = [forme]
94 return formes, lems, tformes
96 def make_ucis_txt_formes_from_tt(corpus, tformes) :
97 ucis_txt = [[treat_forme(forme, tformes) for forme in uci] for uci in corpus.ucis_txt]
99 return [' '.join(uci) for uci in ucis_txt]
101 def get_ucis_from_tt(corpus) :
102 content_split = [tuple(line.split('\t')) for line in corpus.content.splitlines()]
103 #print [i for i, line in enumerate(content_split) if line[0] == u'****']
104 ponct =[u',', u'', u'``', u"''", u':', u'#', u')', '(', u'!', u'?', u';', u'-', u'.', u'...']
105 lformes = [forme for forme in list(set(content_split)) if not forme[0].startswith(u'*') and forme[0] not in ponct]
106 formes, lems, tformes = make_formes_and_lems(lformes)
107 ucis = partition(content_split, [i for i, line in enumerate(content_split) if line[0] == u'****'])
109 ucis = [uci for uci in ucis]
111 indices_max_et = [max([i for i, forme in enumerate(uci) if forme[0].startswith(u'*')]) for uci in ucis]
112 corpus.ucis = [uci[:indices_max_et[i] + 1] for i, uci in enumerate(ucis)]
113 corpus.ucis = [[[et[0] for et in uci],''] for uci in corpus.ucis]
114 corpus.ucis_txt = [uci[indices_max_et[i] + 1:] for i, uci in enumerate(ucis)]
116 corpus.formes = formes
118 return make_ucis_txt_formes_from_tt(corpus, tformes)
120 #with codecs.open(infile, 'r', 'latin1') as f :
121 # #content = [line.split('\t') for line in f]
123 #print time.time() - t1
125 #c1 = content.splitlines()
127 #c1s = [val.split('\t') for val in c1]
131 #sc1 = [val.split('\t') for val in sc1]
134 #formes = [val for val in sc1 if not val[0].isdigit()]
137 #sformes = [val[0] for val in formes]
142 #def make_dicts(inlist) :
145 # for i, forme in enumerate(inlist) :
146 # if tuple(forme) in tot :
147 # tot[tuple(forme)].append(i)
149 # tot[tuple(forme)] = [i]
150 # if forme[1] in totgram :
151 # totgram[forme[1]] += 1
153 # totgram[forme[1]] = 1
154 # return tot, totgram
155 #tot, totgram = make_dicts(c1s)
156 #print 'dico', time.time() - t4
159 # key_file = '/home/pierre/fac/cablegate/keys_english.txt'
160 # with open(key_file, 'r') as f :
162 # keys = keys.splitlines()
163 # keys = [line.split('\t') for line in keys]
168 #kact = [key[0] for key in keys if key[2] == '1']
169 #ksup = [key[0] for key in keys if key[2] == '2']
171 #actives = [[len(tot[forme]), forme[0], forme[1], forme[2]] for forme in tot if forme[1] in kact and len(tot[forme]) > 3]
173 #supps = [[len(tot[forme]), forme[0], forme[1], forme[2]] for forme in tot if forme[1] in ksup and len(tot[forme]) > 3]
176 #words = [[len(tot[word]), word[0], word[1], word[2]] for word in tot]
178 ##hapax = [word for word in words if word[3] == 1]
180 #def print_list(thelist, fileout) :
181 # with open(fileout, 'w') as f :
182 # f.write('\n'.join(['\t'.join(['\t'.join(list(line[1:])), `line[0]`]) for line in thelist]).encode('latin1'))
183 #print_list(words, word_list)
184 #print_list(actives, actives_list)
185 #print_list(supps, supps_list)
186 #print time.time() - t4
190 # return [val.tolist() for val in numpy.split(numpy.array(c1s),[i for i, line in enumerate(c1s) if line[0] == u'****'])]
192 ##def make_ucil(c1s) :
193 #from itertools import izip, chain
194 #def partition(alist, indices):
195 # pairs = izip(chain([0], indices), chain(indices, [None]))
196 # return (alist[i:j] for i, j in pairs)
198 #def partition_uci(uci) :
202 # indices = [i for i, forme in enumerate(uci) if forme[0].startswith(u'*')]
203 # pairs = izip(chain([0], indices), chain(indices, [None]))
204 # return (uci[i,j] for i, j in pairs)
206 ##ucis = make_uci(c1s)
208 #ucis = partition(c1s, [i for i, line in enumerate(c1s) if line[0] == u'****'])
209 #print time.time() - t2
214 ## print max([i for i, forme in enumerate(uci) if forme[0].startswith(u'*')])
216 #ucis = [uci for uci in ucis]
218 #indices_max_et = [max([i for i, forme in enumerate(uci) if forme[0].startswith(u'*')]) for uci in ucis]
219 ##ucis2 = [partition_uci(uci) for uci in ucis]
221 #print len(indices_max_et)
224 #etoiles = [uci[:indices_max_et[i] + 1] for i, uci in enumerate(ucis)]
225 #ucis = [uci[indices_max_et[i] + 1:] for i, uci in enumerate(ucis)]
230 ##ucis = [[val for val in uci if val[1] != 'PUN'] for uci in ucis]
231 #ind_sent = [[i for i, forme in enumerate(uci) if forme[1] == 'SENT'] for uci in ucis]
232 #print time.time() - t3
234 #print len(ucis), len(ind_sent)
235 ##inuformes = [i for i,forme in enumerate(sformes) if sformes.count(forme) > 1]
236 ##inuformes = [formes[i] for i, forme in enumerate(sformes) if forme in sformes[i+1:]]
237 ##nonunique = [forme for forme in formes if formes.count(forme) > 1]
239 #split_sents = [[partition(uci, ind_sent[i])] for i, uci in enumerate(ucis)]
240 #PUNCT = [u',', u'', u'``', u"''", u':', u'#', u')', '(', u'!', u'?', u';', u'-', u'.']
241 #split_sents = [[val for sent in uci for val in sent] for uci in split_sents]
242 #split_sents = [[val for sent in uci for val in sent if val[0] not in PUNCT and not val[0].isdigit()] for uci in split_sents]
243 ##for i in range(0,1) :
244 ## for sent in split_sents[i] :
247 ##nuformes = [formes[i] for i in inuformes]