From 70e42bff043fd0db4f7b769f9985ddf7574b9005 Mon Sep 17 00:00:00 2001
From: "yusuke.shinyama.dummy"
 <yusuke.shinyama.dummy@1aa58f4a-7d42-0410-adbc-911cccaed67c>
Date: Tue, 24 Mar 2009 16:26:59 +0000
Subject: [PATCH] encoding bug fixed.

git-svn-id: https://pdfminerr.googlecode.com/svn/trunk/pdfminer@74 1aa58f4a-7d42-0410-adbc-911cccaed67c
---
 README.html       |   7 +-
 pdflib/cmap.py    |  20 +++--
 pdflib/pdffont.py | 221 ++++++++++++++++++++++++++++++++++++----------
 tools/dumppdf.py  |   2 +-
 4 files changed, 197 insertions(+), 53 deletions(-)

diff --git a/README.html b/README.html
index 0e6418b..709476d 100644
--- a/README.html
+++ b/README.html
@@ -14,7 +14,7 @@ Python PDF parser and analyzer
 
 <div align=right class=lastmod>
 <!-- hhmts start -->
-Last Modified: Mon Feb  2 00:01:01 JST 2009
+Last Modified: Wed Mar 25 01:26:33 JST 2009
 <!-- hhmts end -->
 </div>
 
@@ -175,6 +175,10 @@ By default, it extracts texts from all the pages.
 <dt> <code>-c <em>codec</em></code> 
 <dd> Specifies the output codec for non-ASCII texts.
 <p>
+<dt> <code>-w</code> 
+<dd> Split each word into a different chunk in the output.
+This makes the word spacing correctly handled.
+<p>
 <dt> <code>-t <em>type</em></code> 
 <dd> Specifies the output format. The following formats are currently supported.
 <ul>
@@ -250,6 +254,7 @@ no stream header is displayed for the ease of saving it to a file.
 <hr noshade>
 <h2>Changes</h2>
 <ul>
+<li> 2009/03/25: Encoding problems fixed. Word splitting option added.
 <li> 2009/02/01: Various bugfixes. Thanks to Hiroshi Manabe.
 <li> 2009/01/17: Handling a trailer correctly that contains both /XrefStm and /Prev entries.
 <li> 2009/01/10: Handling Type3 font metrics correctly.
diff --git a/pdflib/cmap.py b/pdflib/cmap.py
index bcdd9e9..1c8ab53 100644
--- a/pdflib/cmap.py
+++ b/pdflib/cmap.py
@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-import sys
+import sys, re
 stderr = sys.stderr
 from struct import pack, unpack
 from utils import choplist, nunpack
@@ -15,6 +15,16 @@ except ImportError:
 class CMapError(Exception): pass
 
 
+STRIP_NAME = re.compile(r'[0-9]+')
+def name2unicode(name):
+  from glyphlist import charname2unicode
+  if name in charname2unicode:
+    return charname2unicode[name]
+  m = STRIP_NAME.search(name)
+  if not m: raise KeyError(name)
+  return int(m.group(0))
+
+
 ##  CMap
 ##
 class CMap(object):
@@ -48,10 +58,9 @@ class CMap(object):
     return self
 
   def register_cid2code(self, cid, code):
-    from glyphlist import charname2unicode
     if isinstance(cid, int):
       if isinstance(code, PSLiteral):
-        self.cid2code[cid] = pack('>H', charname2unicode[code.name])
+        self.cid2code[cid] = pack('>H', name2unicode(code.name))
       elif isinstance(code, str):
         self.cid2code[cid] = code
     return self
@@ -352,7 +361,6 @@ class FontMetricsDB(object):
 ##
 class EncodingDB(object):
       
-  from glyphlist import charname2unicode
   from latin_enc import ENCODING
   
   std2unicode = {}
@@ -360,7 +368,7 @@ class EncodingDB(object):
   win2unicode = {}
   pdf2unicode = {}
   for (name,std,mac,win,pdf) in ENCODING:
-    c = unichr(charname2unicode[name])
+    c = unichr(name2unicode(name))
     if std: std2unicode[std] = c
     if mac: mac2unicode[mac] = c
     if win: win2unicode[win] = c
@@ -384,7 +392,7 @@ class EncodingDB(object):
           cid = x
         elif isinstance(x, PSLiteral):
           try:
-            cid2unicode[cid] = unichr(EncodingDB.charname2unicode[x.name])
+            cid2unicode[cid] = unichr(name2unicode(x.name))
           except KeyError:
             pass
           cid += 1
diff --git a/pdflib/pdffont.py b/pdflib/pdffont.py
index 5735f05..d1f2d0d 100644
--- a/pdflib/pdffont.py
+++ b/pdflib/pdffont.py
@@ -15,51 +15,135 @@ from cmap import CMap, CMapDB, CMapParser, FontMetricsDB, EncodingDB
 from utils import apply_matrix_norm, nunpack
 
 
-NIBBLES = ('0','1','2','3','4','5','6','7','8','9','.','e','e-',None,'-')
-def getnum(fp):
-  b0 = ord(fp.read(1))
-  if b0 == 30:
-    s = ''
-    loop = True
-    while loop:
-      b = ord(fp.read(1))
-      for n in (b >> 4, b & 15):
-        if n == 15:
-          loop = False
-        else:
-          s += NIBBLES[n]
-    return float(s)
-  if 32 <= b0 and b0 <= 246:
-    return b0-139
-  b1 = ord(fp.read(1))
-  if 247 <= b0 and b0 <= 250:
-    return ((b0-247)<<8)+b1+108
-  if 251 <= b0 and b0 <= 254:
-    return -((b0-251)<<8)-b1-108
-  b2 = ord(fp.read(1))
-  if 128 <= b1: b1 -= 256
-  if b0 == 28:
-    return b1<<8 | b2
-  return b1<<24 | b2<<16 | unpack('>H',fp.read(2))[0]
-#assert getop(StringIO('\x8b')) == 0
-#assert getop(StringIO('\xef')) == 100
-#assert getop(StringIO('\x27')) == -100
-#assert getop(StringIO('\xfa\x7c')) == 1000
-#assert getop(StringIO('\xfe\x7c')) == -1000
-#assert getop(StringIO('\x1c\x27\x10')) == 10000
-#assert getop(StringIO('\x1c\xd8\xf0')) == -10000
-#assert getop(StringIO('\x1d\x00\x01\x86\xa0')) == 100000
-#assert getop(StringIO('\x1d\xff\xfe\x79\x60')) == -100000
-#assert getop(StringIO('\x1e\xe2\xa2\x5f')) == -2.25
-#assert getop(StringIO('\x1e\x0a\x14\x05\x41\xc3\xff')) == 0.140541e-3
-
-
 ##  CFFFont
 ##  (Format specified in Adobe Technical Note: #5176
 ##   "The Compact Font Format Specification")
 ##
+NIBBLES = ('0','1','2','3','4','5','6','7','8','9','.','e','e-',None,'-')
+def getdict(data):
+  d = {}
+  fp = StringIO(data)
+  stack = []
+  while 1:
+    c = fp.read(1)
+    if not c: break
+    b0 = ord(c)
+    if b0 <= 21:
+      d[b0] = stack
+      stack = []
+      continue
+    if b0 == 30:
+      s = ''
+      loop = True
+      while loop:
+        b = ord(fp.read(1))
+        for n in (b >> 4, b & 15):
+          if n == 15:
+            loop = False
+          else:
+            s += NIBBLES[n]
+      value = float(s)
+    elif 32 <= b0 and b0 <= 246:
+      value = b0-139
+    else:
+      b1 = ord(fp.read(1))
+      if 247 <= b0 and b0 <= 250:
+        value = ((b0-247)<<8)+b1+108
+      elif 251 <= b0 and b0 <= 254:
+        value = -((b0-251)<<8)-b1-108
+      else:
+        b2 = ord(fp.read(1))
+        if 128 <= b1: b1 -= 256
+        if b0 == 28:
+          value = b1<<8 | b2
+        else:
+          value = b1<<24 | b2<<16 | unpack('>H', fp.read(2))[0]
+    stack.append(value)
+  return d
+
 class CFFFont(object):
 
+  STANDARD_STRINGS = (
+    '.notdef', 'space', 'exclam', 'quotedbl', 'numbersign',
+    'dollar', 'percent', 'ampersand', 'quoteright', 'parenleft',
+    'parenright', 'asterisk', 'plus', 'comma', 'hyphen', 'period',
+    'slash', 'zero', 'one', 'two', 'three', 'four', 'five', 'six',
+    'seven', 'eight', 'nine', 'colon', 'semicolon', 'less', 'equal',
+    'greater', 'question', 'at', 'A', 'B', 'C', 'D', 'E', 'F', 'G',
+    'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T',
+    'U', 'V', 'W', 'X', 'Y', 'Z', 'bracketleft', 'backslash',
+    'bracketright', 'asciicircum', 'underscore', 'quoteleft', 'a',
+    'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n',
+    'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+    'braceleft', 'bar', 'braceright', 'asciitilde', 'exclamdown',
+    'cent', 'sterling', 'fraction', 'yen', 'florin', 'section',
+    'currency', 'quotesingle', 'quotedblleft', 'guillemotleft',
+    'guilsinglleft', 'guilsinglright', 'fi', 'fl', 'endash',
+    'dagger', 'daggerdbl', 'periodcentered', 'paragraph', 'bullet',
+    'quotesinglbase', 'quotedblbase', 'quotedblright',
+    'guillemotright', 'ellipsis', 'perthousand', 'questiondown',
+    'grave', 'acute', 'circumflex', 'tilde', 'macron', 'breve',
+    'dotaccent', 'dieresis', 'ring', 'cedilla', 'hungarumlaut',
+    'ogonek', 'caron', 'emdash', 'AE', 'ordfeminine', 'Lslash',
+    'Oslash', 'OE', 'ordmasculine', 'ae', 'dotlessi', 'lslash',
+    'oslash', 'oe', 'germandbls', 'onesuperior', 'logicalnot', 'mu',
+    'trademark', 'Eth', 'onehalf', 'plusminus', 'Thorn',
+    'onequarter', 'divide', 'brokenbar', 'degree', 'thorn',
+    'threequarters', 'twosuperior', 'registered', 'minus', 'eth',
+    'multiply', 'threesuperior', 'copyright', 'Aacute',
+    'Acircumflex', 'Adieresis', 'Agrave', 'Aring', 'Atilde',
+    'Ccedilla', 'Eacute', 'Ecircumflex', 'Edieresis', 'Egrave',
+    'Iacute', 'Icircumflex', 'Idieresis', 'Igrave', 'Ntilde',
+    'Oacute', 'Ocircumflex', 'Odieresis', 'Ograve', 'Otilde',
+    'Scaron', 'Uacute', 'Ucircumflex', 'Udieresis', 'Ugrave',
+    'Yacute', 'Ydieresis', 'Zcaron', 'aacute', 'acircumflex',
+    'adieresis', 'agrave', 'aring', 'atilde', 'ccedilla', 'eacute',
+    'ecircumflex', 'edieresis', 'egrave', 'iacute', 'icircumflex',
+    'idieresis', 'igrave', 'ntilde', 'oacute', 'ocircumflex',
+    'odieresis', 'ograve', 'otilde', 'scaron', 'uacute',
+    'ucircumflex', 'udieresis', 'ugrave', 'yacute', 'ydieresis',
+    'zcaron', 'exclamsmall', 'Hungarumlautsmall', 'dollaroldstyle',
+    'dollarsuperior', 'ampersandsmall', 'Acutesmall',
+    'parenleftsuperior', 'parenrightsuperior', 'twodotenleader',
+    'onedotenleader', 'zerooldstyle', 'oneoldstyle', 'twooldstyle',
+    'threeoldstyle', 'fouroldstyle', 'fiveoldstyle', 'sixoldstyle',
+    'sevenoldstyle', 'eightoldstyle', 'nineoldstyle',
+    'commasuperior', 'threequartersemdash', 'periodsuperior',
+    'questionsmall', 'asuperior', 'bsuperior', 'centsuperior',
+    'dsuperior', 'esuperior', 'isuperior', 'lsuperior', 'msuperior',
+    'nsuperior', 'osuperior', 'rsuperior', 'ssuperior', 'tsuperior',
+    'ff', 'ffi', 'ffl', 'parenleftinferior', 'parenrightinferior',
+    'Circumflexsmall', 'hyphensuperior', 'Gravesmall', 'Asmall',
+    'Bsmall', 'Csmall', 'Dsmall', 'Esmall', 'Fsmall', 'Gsmall',
+    'Hsmall', 'Ismall', 'Jsmall', 'Ksmall', 'Lsmall', 'Msmall',
+    'Nsmall', 'Osmall', 'Psmall', 'Qsmall', 'Rsmall', 'Ssmall',
+    'Tsmall', 'Usmall', 'Vsmall', 'Wsmall', 'Xsmall', 'Ysmall',
+    'Zsmall', 'colonmonetary', 'onefitted', 'rupiah', 'Tildesmall',
+    'exclamdownsmall', 'centoldstyle', 'Lslashsmall', 'Scaronsmall',
+    'Zcaronsmall', 'Dieresissmall', 'Brevesmall', 'Caronsmall',
+    'Dotaccentsmall', 'Macronsmall', 'figuredash', 'hypheninferior',
+    'Ogoneksmall', 'Ringsmall', 'Cedillasmall', 'questiondownsmall',
+    'oneeighth', 'threeeighths', 'fiveeighths', 'seveneighths',
+    'onethird', 'twothirds', 'zerosuperior', 'foursuperior',
+    'fivesuperior', 'sixsuperior', 'sevensuperior', 'eightsuperior',
+    'ninesuperior', 'zeroinferior', 'oneinferior', 'twoinferior',
+    'threeinferior', 'fourinferior', 'fiveinferior', 'sixinferior',
+    'seveninferior', 'eightinferior', 'nineinferior',
+    'centinferior', 'dollarinferior', 'periodinferior',
+    'commainferior', 'Agravesmall', 'Aacutesmall',
+    'Acircumflexsmall', 'Atildesmall', 'Adieresissmall',
+    'Aringsmall', 'AEsmall', 'Ccedillasmall', 'Egravesmall',
+    'Eacutesmall', 'Ecircumflexsmall', 'Edieresissmall',
+    'Igravesmall', 'Iacutesmall', 'Icircumflexsmall',
+    'Idieresissmall', 'Ethsmall', 'Ntildesmall', 'Ogravesmall',
+    'Oacutesmall', 'Ocircumflexsmall', 'Otildesmall',
+    'Odieresissmall', 'OEsmall', 'Oslashsmall', 'Ugravesmall',
+    'Uacutesmall', 'Ucircumflexsmall', 'Udieresissmall',
+    'Yacutesmall', 'Thornsmall', 'Ydieresissmall', '001.000',
+    '001.001', '001.002', '001.003', 'Black', 'Bold', 'Book',
+    'Light', 'Medium', 'Regular', 'Roman', 'Semibold',
+    )
+
   class INDEX(object):
     
     def __init__(self, fp):
@@ -82,11 +166,13 @@ class CFFFont(object):
       self.fp.seek(self.base+self.offsets[i])
       return self.fp.read(self.offsets[i+1]-self.offsets[i])
 
-  def __init__(self, name, fp):
-    self.name = name
-    self.fp = fp
+    def __iter__(self):
+      return iter( self[i] for i in xrange(len(self)) )
+
+  def __init__(self, fp0):
+    self.fp = fp0
     # Header
-    (_major,_minor,hdrsize,self.offsize) = unpack('BBBB', fp.read(4))
+    (_major,_minor,hdrsize,self.offsize) = unpack('BBBB', self.fp.read(4))
     self.fp.read(hdrsize-4)
     # Name INDEX
     self.name_index = self.INDEX(self.fp)
@@ -96,11 +182,55 @@ class CFFFont(object):
     self.string_index = self.INDEX(self.fp)
     # Global Subr INDEX
     self.subr_index = self.INDEX(self.fp)
+    # Top DICT DATA
+    self.top_dict = getdict(self.dict_index[0])
+    (charset_pos,) = self.top_dict.get(15, 0)
+    (encoding_pos,) = self.top_dict.get(16, 0)
+    (charstring_pos,) = self.top_dict.get(17, 0)
+    # CharStrings
+    self.fp.seek(charstring_pos)
+    self.charstring = self.INDEX(self.fp)
+    self.nglyphs = len(self.charstring)
     # Encodings
+    self.code2gid = {}
+    self.gid2code = {}
+    self.fp.seek(encoding_pos)
+    format = self.fp.read(1)
+    if format == '\x00':
+      # Format 0
+      (n,) = unpack('B', self.fp.read(1))
+      for (code,gid) in enumerate(unpack('B'*n, self.fp.read(n))):
+        self.code2gid[code] = gid
+        self.gid2code[gid] = code
+    else:
+      # Format 1
+      assert 0
     # Charsets
+    self.name2gid = {}
+    self.gid2name = {}
+    self.fp.seek(charset_pos)
+    format = self.fp.read(1)
+    if format == '\x00':
+      # Format 0
+      n = self.nglyphs-1
+      for (gid,sid) in enumerate(unpack('>'+'H'*n, self.fp.read(2*n))):
+        gid += 1
+        name = self.getstr(sid)
+        self.name2gid[name] = gid
+        self.gid2name[gid] = name
+    else:
+      # Format 1
+      assert 0
+    #print self.code2gid
+    #print self.name2gid
+    #assert 0
     return
 
-  
+  def getstr(self, sid):
+    if sid < len(self.STANDARD_STRINGS):
+      return self.STANDARD_STRINGS[sid]
+    return self.string_index[sid-len(self.STANDARD_STRINGS)]
+
   
 ##  TrueTypeFont
 ##
@@ -184,6 +314,7 @@ class PDFFontError(PDFException): pass
 class PDFUnicodeNotDefined(PDFFontError): pass
 
 LITERAL_STANDARD_ENCODING = PSLiteralTable.intern('StandardEncoding')
+LITERAL_TYPE1C = PSLiteralTable.intern('Type1C')
 
 
 # PDFFont
@@ -449,7 +580,7 @@ class PDFCIDFont(PDFFont):
 def main(argv):
   for fname in argv[1:]:
     fp = file(fname, 'rb')
-    CFFFont(fname, fp)
+    CFFFont(fp)
     fp.close()
   return
 if __name__ == '__main__': sys.exit(main(sys.argv))
diff --git a/tools/dumppdf.py b/tools/dumppdf.py
index 8eae1fe..61b2aef 100755
--- a/tools/dumppdf.py
+++ b/tools/dumppdf.py
@@ -13,7 +13,7 @@ stdout = sys.stdout
 stderr = sys.stderr
 
 
-ESC_PAT = re.compile(r'[\000-\037&<>\042\047\134\177-\377]')
+ESC_PAT = re.compile(r'[\000-\037&<>()\042\047\134\177-\377]')
 def esc(s):
   return ESC_PAT.sub(lambda m:'&#%d;' % ord(m.group(0)), s)