1 # -*- coding: utf-8 -*-
2 #Author: Pierre Ratinaud
3 #Copyright (c) 2014 Pierre Ratinaud
7 #from BeautifulSoup import BeautifulSoup
10 from HTMLParser import HTMLParser
13 mois = {u'janvier' : '01',
48 return [`year`, month, '%02d' % day]
54 return [year, month, day]
57 # create a subclass and override the handler methods
58 class MyHTMLParser(HTMLParser):
59 def handle_starttag(self, tag, attrs):
60 #print "Encountered a start tag:", tag
63 if attrs[0][1] == 'DocPublicationName' :
64 #print 'DocPublicationName'
66 self.currentattr = 'DocPublicationName'
67 elif attrs[0][1] == 'DocHeader' :
69 self.currentattr = 'DocHeader'
70 elif attrs[0][1] in ['TitreArticleVisu', 'titreArticleVisu'] :
71 self.outfile.write('\n\n')
72 self.meta.append('\n')
73 self.outfile.write(' '.join(self.meta).encode('utf8', errors='replace'))
76 self.currentattr = 'TitreArticleVisu'
77 elif attrs[0][1] == 'PubliC_lblNodoc' :
78 self.currentattr = 'PubliC_lblNodoc'
80 self.currentattr = None
83 if attrs[0][1] == 'publiC-lblNodoc' :
84 self.currentattr = 'PubliC_lblNodoc'
87 if attrs[0][1] == 'titreArticleVisu' :
88 # self.outfile.write('\n\n')
89 # self.meta.append('\n')
90 # self.outfile.write(' '.join(self.meta).encode('utf8', errors='replace'))
91 # self.meta = [u'****']
93 self.currentattr = 'TitreArticleVisu'
95 def handle_endtag(self, tag):
97 #print "Encountered an end tag :", tag
98 def handle_data(self, data):
99 if self.currentattr == 'DocPublicationName' :
101 PublicationName = data.strip().replace(' ', '_').replace('(','').replace(')','').replace('-','').replace('.','').replace('/','').replace("'",'').replace(';', '').replace(':', '').replace(u'·','').lower()
102 PublicationName = PublicationName.split(',')[0]
103 self.meta.append(u'*source_' + PublicationName)
104 self.currentattr = None
105 # elif self.currentattr == 'DocHeader' :
106 # date = finddate(data)
107 # if date is not None :
108 # self.meta += [u'*date_' + '-'.join(date), u'*am_' + '-'.join(date[0:2]), u'*annee_' + date[0]]
109 elif self.currentattr == 'TitreArticleVisu' :
111 if data.startswith(u'©') :
112 self.currentattr = None
114 self.content.append(' '.join(data.replace('\n', ' ').split()) + ' ')
115 #self.outfile.write(' '.join(data.replace('\n', ' ').split()).encode('utf8', errors='replace') + ' ')
116 elif self.currentattr == 'PubliC_lblNodoc' :
118 date = makedate(date)
119 self.meta += [u'*date_' + '-'.join(date), u'*am_' + '-'.join(date[0:2]), u'*annee_' + date[0]]
120 self.meta.append('\n')
121 self.outfile.write('\n\n')
122 self.outfile.write(' '.join(self.meta).encode('utf8', errors='replace'))
123 self.outfile.write(' '.join(self.content).encode('utf8'))
125 self.meta = [u'****']
127 self.currentattr = None
129 def doinit(self, outfile):
130 self.currentattr = None
131 self.meta = [u'****']
134 self.outfile = outfile
138 def ParseEuropress(txtdir, fileout, encodage_in, encodage_out) :
140 for root, subfolders, subfiles in os.walk(txtdir) :
141 nf = [os.path.join(root, f) for f in subfiles if f.split('.')[-1] in ['html', 'HTML'] ]
147 parser = MyHTMLParser()
148 with open(fileout,'w') as outf :
152 with codecs.open(f, 'rU', encodage_in) as infile :
153 content = infile.read()
154 content = HTMLParser().unescape(content)
159 #ParseEuropress('/home/pierre/fac/etudiant/DeNadai/corpus_loi_travail',
160 # '/home/pierre/fac/etudiant/DeNadai/corpus_loi_W.txt', 'utf8', 'utf8')