#!/usr/bin/env python
import sys
from pdfparser import PDFDocument, PDFParser, PDFPasswordIncorrect
from pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfdevice import PDFDevice, PDFPageAggregator
from layout import Page, FigureItem, TextItem, cluster_textobjs
from pdffont import PDFUnicodeNotDefined
from cmap import CMapDB
def enc(x, codec):
x = x.replace('&','&').replace('>','>').replace('<','<').replace('"','"')
return x.encode(codec, 'xmlcharrefreplace')
def encprops(props, codec):
if not props: return ''
return ''.join( ' %s="%s"' % (enc(k,codec), enc(str(v),codec)) for (k,v) in sorted(props.iteritems()) )
def get_textobjs(item, r=None):
if r == None: r = []
if isinstance(item, TextItem):
r.append(item)
elif isinstance(item, Page):
for child in item.objs:
get_textobjs(child, r)
return r
## PDFConverter
class PDFConverter(PDFPageAggregator):
def __init__(self, rsrc, outfp, codec='ascii'):
PDFPageAggregator.__init__(self, rsrc)
self.outfp = outfp
self.codec = codec
return
## SGMLConverter
##
class SGMLConverter(PDFConverter):
def end_page(self, page):
page = PDFConverter.end_page(self, page)
def f(item):
bbox = '%.3f,%.3f,%.3f,%.3f' % item.bbox
if isinstance(item, FigureItem):
self.outfp.write('\n' % (item.id, bbox))
for child in item.objs:
f(child)
self.outfp.write('\n')
elif isinstance(item, TextItem):
self.outfp.write('' %
(enc(item.font.fontname, self.codec), item.vertical, bbox, item.fontsize))
self.outfp.write(enc(item.text, self.codec))
self.outfp.write('\n')
bbox = '%.3f,%.3f,%.3f,%.3f' % page.bbox
self.outfp.write('\n' %
(page.id, bbox, page.rotate))
for child in page.objs:
f(child)
self.outfp.write('\n')
return
## HTMLConverter
##
class HTMLConverter(PDFConverter):
def __init__(self, rsrc, outfp, codec='utf-8', pagenum=True, pagepad=50, scale=1, cluster_margin=None):
PDFConverter.__init__(self, rsrc, outfp, codec=codec)
self.pagenum = pagenum
self.pagepad = pagepad
self.scale = scale
self.outfp.write('\n' % self.codec)
self.outfp.write('\n')
self.yoffset = self.pagepad
self.cluster_margin = cluster_margin
self.show_text_border = False
return
def end_page(self, page):
page = PDFConverter.end_page(self, page)
self.yoffset += page.y1
if self.pagenum:
self.outfp.write('' %
((self.yoffset-page.y1)*self.scale, page.id, page.id))
self.outfp.write('\n' %
(page.x0*self.scale, (self.yoffset-page.y1)*self.scale,
page.width*self.scale, page.height*self.scale))
def draw(item):
if isinstance(item, FigureItem):
for child in item.objs:
draw(child)
elif isinstance(item, TextItem):
if item.vertical:
wmode = 'tb-rl'
else:
wmode = 'lr-tb'
self.outfp.write('' %
(wmode, item.x0*self.scale, (self.yoffset-item.y1)*self.scale,
item.fontsize*self.scale))
self.outfp.write(enc(item.text, self.codec))
self.outfp.write('\n')
if self.show_text_border:
self.outfp.write('\n' %
(item.x0*self.scale, (self.yoffset-item.y1)*self.scale,
item.width*self.scale, self.height*self.scale))
for child in page.objs:
draw(child)
if self.cluster_margin:
clusters = cluster_textobjs(get_textobjs(page), self.cluster_margin)
for textbox in clusters:
self.outfp.write('\n' %
(textbox.x0*self.scale, (self.yoffset-textbox.y1)*self.scale,
textbox.width*self.scale, textbox.height*self.scale))
self.yoffset += self.pagepad
return
def close(self):
self.outfp.write('Page: %s
\n' %
', '.join('%s' % (i,i) for i in xrange(1,self.pageno)))
self.outfp.write('\n')
return
## TextConverter
##
class TextConverter(PDFConverter):
def __init__(self, rsrc, outfp, codec='utf-8', pagenum=False, cluster_margin=None):
PDFConverter.__init__(self, rsrc, outfp, codec=codec)
self.pagenum = pagenum
if cluster_margin == None:
cluster_margin = 0.5
self.cluster_margin = cluster_margin
self.word_margin = 0.2
return
def end_page(self, page):
page = PDFConverter.end_page(self, page)
if self.pagenum:
self.outfp.write('Page %d\n' % page.id)
if self.cluster_margin:
textobjs = get_textobjs(page)
clusters = cluster_textobjs(textobjs, self.cluster_margin)
for textbox in clusters:
for line in textbox.lines(self.word_margin):
self.outfp.write(line.encode(self.codec, 'replace')+'\n')
self.outfp.write('\n')
else:
for obj in page.objs:
if isinstance(obj, TextItem):
self.outfp.write(obj.text.encode(self.codec, 'replace'))
self.outfp.write('\n')
self.outfp.write('\f')
return
def close(self):
return
## TagExtractor
##
class TagExtractor(PDFDevice):
def __init__(self, rsrc, outfp, codec='utf-8'):
PDFDevice.__init__(self, rsrc)
self.outfp = outfp
self.codec = codec
self.pageno = 0
self.tag = None
return
def render_image(self, stream, size, matrix):
return
def render_string(self, textstate, textmatrix, seq):
font = textstate.font
text = ''
for x in seq:
if not isinstance(x, str): continue
chars = font.decode(x)
for cid in chars:
try:
char = font.to_unicode(cid)
text += char
except PDFUnicodeNotDefined, e:
pass
self.outfp.write(enc(text, self.codec))
return
def begin_page(self, page):
(x0, y0, x1, y1) = page.mediabox
bbox = '%.3f,%.3f,%.3f,%.3f' % (x0, y0, x1, y1)
self.outfp.write('' %
(self.pageno, bbox, page.rotate))
return
def end_page(self, page):
self.outfp.write('\n')
self.pageno += 1
return
def begin_tag(self, tag, props=None):
self.outfp.write('<%s%s>' % (enc(tag.name, self.codec), encprops(props, self.codec)))
self.tag = tag
return
def end_tag(self):
assert self.tag
self.outfp.write('' % enc(self.tag.name, self.codec))
self.tag = None
return
def do_tag(self, tag, props=None):
self.outfp.write('<%s%s/>' % (enc(tag.name, self.codec), encprops(props, self.codec)))
return
# pdf2txt
class TextExtractionNotAllowed(RuntimeError): pass
def convert(rsrc, device, fname, pagenos=None, maxpages=0, password=''):
doc = PDFDocument()
fp = file(fname, 'rb')
parser = PDFParser(doc, fp)
try:
doc.initialize(password)
except PDFPasswordIncorrect:
raise TextExtractionNotAllowed('Incorrect password')
if not doc.is_extractable:
raise TextExtractionNotAllowed('Text extraction is not allowed: %r' % fname)
interpreter = PDFPageInterpreter(rsrc, device)
for (pageno,page) in enumerate(doc.get_pages()):
if pagenos and (pageno not in pagenos): continue
interpreter.process_page(page)
if maxpages and maxpages <= pageno+1: break
device.close()
fp.close()
return
# main
def main(argv):
import getopt
def usage():
print 'usage: %s [-d] [-p pagenos] [-P password] [-c codec] [-w] [-t text|html|sgml|tag] [-o output] file ...' % argv[0]
return 100
try:
(opts, args) = getopt.getopt(argv[1:], 'dp:P:c:T:t:o:C:D:m:w')
except getopt.GetoptError:
return usage()
if not args: return usage()
debug = 0
cmapdir = 'CMap'
cdbcmapdir = 'CDBCMap'
codec = 'ascii'
pagenos = set()
maxpages = 0
outtype = 'html'
password = ''
pagenum = True
splitwords = False
cluster_margin = None
outfp = sys.stdout
for (k, v) in opts:
if k == '-d': debug += 1
elif k == '-p': pagenos.update( int(x)-1 for x in v.split(',') )
elif k == '-P': password = v
elif k == '-c': codec = v
elif k == '-m': maxpages = int(v)
elif k == '-C': cmapdir = v
elif k == '-D': cdbcmapdir = v
elif k == '-T': cluster_margin = float(v)
elif k == '-t': outtype = v
elif k == '-o': outfp = file(v, 'wb')
elif k == '-w': splitwords = True
#
CMapDB.debug = debug
PDFResourceManager.debug = debug
PDFDocument.debug = debug
PDFParser.debug = debug
PDFPageInterpreter.debug = debug
#
CMapDB.initialize(cmapdir, cdbcmapdir)
rsrc = PDFResourceManager()
if outtype == 'sgml':
device = SGMLConverter(rsrc, outfp, codec=codec)
elif outtype == 'html':
device = HTMLConverter(rsrc, outfp, codec=codec, cluster_margin=cluster_margin)
elif outtype == 'text':
device = TextConverter(rsrc, outfp, codec=codec, cluster_margin=cluster_margin)
elif outtype == 'tag':
device = TagExtractor(rsrc, outfp, codec=codec)
else:
return usage()
for fname in args:
convert(rsrc, device, fname, pagenos,
maxpages=maxpages, password=password)
return
if __name__ == '__main__': sys.exit(main(sys.argv))