From 5115a4aed60057acdfea244dac1f2b057c69e6ae Mon Sep 17 00:00:00 2001 From: Apprentice Alf Date: Fri, 27 Nov 2009 20:51:55 +0000 Subject: [PATCH] ereader2html 0.07a last version before the split into erdr2pml and xpml2xhtml, although there were more versions of the combined script. --- eReader_Tools/lib/ereader2html.py | 282 +++++++++++++----------------- 1 file changed, 124 insertions(+), 158 deletions(-) diff --git a/eReader_Tools/lib/ereader2html.py b/eReader_Tools/lib/ereader2html.py index aacd962..6103dd0 100644 --- a/eReader_Tools/lib/ereader2html.py +++ b/eReader_Tools/lib/ereader2html.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: ascii -*- # vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab # # This is a python script. You need a Python interpreter to run it. @@ -37,17 +36,18 @@ # # TODO run this through a profiler - speed increases so far was from # applying "quick known fixes", added (commented out) cprofiler call -# 0.05 - Support for more things in type 272 +# 0.05 - Improved type 272 support for sidebars, links, chapters, metainfo, etc # 0.06 - Merge of 0.04 and 0.05. Improved HTML output # Placed images in subfolder, so that it's possible to just # drop the book.pml file onto DropBook to make an unencrypted # copy of the eReader file. # Using that with Calibre works a lot better than the HTML # conversion in this code. +# 0.07 - Further Improved type 272 support for sidebars with all earlier fixes +# 0.07a - Fixed some typos -__version__='0.06' -DEBUG = 0 +__version__='0.07' # Import Psyco if available try: @@ -82,8 +82,8 @@ logging.basicConfig() write_pml=False -ECB = 0 -CBC = 1 +ECB = 0 +CBC = 1 class Des(object): __pc1 = [56, 48, 40, 32, 24, 16, 8, 0, 57, 49, 41, 33, 25, 17, 9, 1, 58, 50, 42, 34, 26, 18, 10, 2, 59, 51, 43, 35, @@ -91,13 +91,13 @@ class Des(object): 13, 5, 60, 52, 44, 36, 28, 20, 12, 4, 27, 19, 11, 3] __left_rotations = [1, 1, 2, 2, 2, 2, 2, 2, 1, 2, 2, 2, 2, 2, 2, 1] __pc2 = [13, 16, 10, 23, 0, 4,2, 27, 14, 5, 20, 9, - 22, 18, 11, 3, 25, 7, 15, 6, 26, 19, 12, 1, - 40, 51, 30, 36, 46, 54, 29, 39, 50, 44, 32, 47, - 43, 48, 38, 55, 33, 52, 45, 41, 49, 35, 28, 31] - __ip = [57, 49, 41, 33, 25, 17, 9, 1, 59, 51, 43, 35, 27, 19, 11, 3, - 61, 53, 45, 37, 29, 21, 13, 5, 63, 55, 47, 39, 31, 23, 15, 7, - 56, 48, 40, 32, 24, 16, 8, 0, 58, 50, 42, 34, 26, 18, 10, 2, - 60, 52, 44, 36, 28, 20, 12, 4, 62, 54, 46, 38, 30, 22, 14, 6] + 22, 18, 11, 3, 25, 7, 15, 6, 26, 19, 12, 1, + 40, 51, 30, 36, 46, 54, 29, 39, 50, 44, 32, 47, + 43, 48, 38, 55, 33, 52, 45, 41, 49, 35, 28, 31] + __ip = [57, 49, 41, 33, 25, 17, 9, 1, 59, 51, 43, 35, 27, 19, 11, 3, + 61, 53, 45, 37, 29, 21, 13, 5, 63, 55, 47, 39, 31, 23, 15, 7, + 56, 48, 40, 32, 24, 16, 8, 0, 58, 50, 42, 34, 26, 18, 10, 2, + 60, 52, 44, 36, 28, 20, 12, 4, 62, 54, 46, 38, 30, 22, 14, 6] __expansion_table = [31, 0, 1, 2, 3, 4, 3, 4, 5, 6, 7, 8, 7, 8, 9, 10, 11, 12,11, 12, 13, 14, 15, 16, 15, 16, 17, 18, 19, 20,19, 20, 21, 22, 23, 24, @@ -141,8 +141,8 @@ class Des(object): 35, 3, 43, 11, 51, 19, 59, 27,34, 2, 42, 10, 50, 18, 58, 26, 33, 1, 41, 9, 49, 17, 57, 25,32, 0, 40, 8, 48, 16, 56, 24] # Type of crypting being done - ENCRYPT = 0x00 - DECRYPT = 0x01 + ENCRYPT = 0x00 + DECRYPT = 0x01 def __init__(self, key, mode=ECB, IV=None): if len(key) != 8: raise ValueError("Invalid DES key size. Key must be exactly 8 bytes long.") @@ -154,7 +154,7 @@ class Des(object): self.setIV(IV) self.L = [] self.R = [] - self.Kn = [ [0] * 48 ] * 16 # 16 48-bit keys (K1 - K16) + self.Kn = [ [0] * 48 ] * 16 # 16 48-bit keys (K1 - K16) self.final = [] self.setKey(key) def getKey(self): @@ -303,20 +303,13 @@ class Sectionizer(object): def __init__(self, filename, ident): self.contents = file(filename, 'rb').read() self.header = self.contents[0:72] - if DEBUG: - file("header.dat", 'wb').write(self.header) - self.num_sections, = struct.unpack('>H', self.contents[76:78]) - if DEBUG: - print "number of sections: %d" % self.num_sections if self.header[0x3C:0x3C+8] != ident: raise ValueError('Invalid file format') self.sections = [] for i in xrange(self.num_sections): offset, a1,a2,a3,a4 = struct.unpack('>LBBBB', self.contents[78+i*8:78+i*8+8]) flags, val = a1, a2<<16|a3<<8|a4 - if DEBUG: - print "section %d offset %d" % ( i, offset ) self.sections.append( (offset, flags, val) ) def loadSection(self, section): if section + 1 == self.num_sections: @@ -336,7 +329,7 @@ def sanitizeFileName(s): def fixKey(key): def fixByte(b): return b ^ ((b ^ (b<<1) ^ (b<<2) ^ (b<<3) ^ (b<<4) ^ (b<<5) ^ (b<<6) ^ (b<<7) ^ 0x80) & 0x80) - return "".join([chr(fixByte(ord(a))) for a in key]) + return "".join([chr(fixByte(ord(a))) for a in key]) def deXOR(text, sp, table): r='' @@ -352,9 +345,6 @@ class EreaderProcessor(object): def __init__(self, section_reader, username, creditcard): self.section_reader = section_reader data = section_reader(0) - self.data0 = data - if DEBUG: - file("data0.dat", 'wb').write(self.data0) version, = struct.unpack('>H', data[0:2]) self.version = version logging.info('eReader file format version %s', version) @@ -362,62 +352,44 @@ class EreaderProcessor(object): raise ValueError('incorrect eReader version %d (error 1)' % version) data = section_reader(1) self.data = data - # first key is used on last 8 bytes of data to get cookie_shuf and cookie_size - self.first_key = data[0:8] - des = Des(fixKey(self.first_key)) - self.end_key = des.decrypt(data[-8:]) - cookie_shuf, cookie_size = struct.unpack('>LL', self.end_key) + des = Des(fixKey(data[0:8])) + cookie_shuf, cookie_size = struct.unpack('>LL', des.decrypt(data[-8:])) if cookie_shuf < 3 or cookie_shuf > 0x14 or cookie_size < 0xf0 or cookie_size > 0x200: raise ValueError('incorrect eReader version (error 2)') - # first key is also used to decrypt all remaining bytes in data into their shuffled version - cookie = des.decrypt(data[-cookie_size:]) - - # now unshuffle it + input = des.decrypt(data[-cookie_size:]) def unshuff(data, shuf): r = [''] * len(data) j = 0 for i in xrange(len(data)): j = (j + shuf) % len(data) r[j] = data[i] - assert len("".join(r)) == len(data) + assert len("".join(r)) == len(data) return "".join(r) - r = unshuff(cookie[0:-8], cookie_shuf) - if DEBUG: - file("mainrecord.dat", 'wb').write(r) - + r = unshuff(input[0:-8], cookie_shuf) + def fixUsername(s): r = '' for c in s.lower(): if (c >= 'a' and c <= 'z' or c >= '0' and c <= '9'): r += c return r - - # second key is made of of user private info i.e. name and crediticard info + user_key = struct.pack('>LL', binascii.crc32(fixUsername(username)) & 0xffffffff, binascii.crc32(creditcard[-8:])& 0xffffffff) - self.user_key = user_key - - # the unshuffled data is examined to find record information drm_sub_version = struct.unpack('>H', r[0:2])[0] self.num_text_pages = struct.unpack('>H', r[2:4])[0] - 1 self.num_image_pages = struct.unpack('>H', r[26:26+2])[0] self.first_image_page = struct.unpack('>H', r[24:24+2])[0] - - if version == 272: + if self.version == 272: self.num_chapter_pages = struct.unpack('>H', r[22:22+2])[0] self.first_chapter_page = struct.unpack('>H', r[20:20+2])[0] - self.num_link_pages = struct.unpack('>H', r[30:30+2])[0] self.first_link_page = struct.unpack('>H', r[28:28+2])[0] - - self.num_bkinfo_pages = struct.unpack('>H', r[34:34+2])[0] - self.first_bkinfo_page = struct.unpack('>H', r[32:32+2])[0] - + self.num_bookinfo_pages = struct.unpack('>H', r[34:34+2])[0] + self.first_bookinfo_page = struct.unpack('>H', r[32:32+2])[0] self.num_footnote_pages = struct.unpack('>H', r[46:46+2])[0] self.first_footnote_page = struct.unpack('>H', r[44:44+2])[0] - self.num_xtextsize_pages = struct.unpack('>H', r[54:54+2])[0] self.first_xtextsize_page = struct.unpack('>H', r[52:52+2])[0] - self.num_sidebar_pages = struct.unpack('>H', r[38:38+2])[0] self.first_sidebar_page = struct.unpack('>H', r[36:36+2])[0] @@ -429,51 +401,25 @@ class EreaderProcessor(object): else: self.num_chapter_pages = 0 self.num_link_pages = 0 - self.num_bkinfo_pages = 0 + self.num_bookinfo_pages = 0 self.num_footnote_pages = 0 self.num_xtextsize_pages = 0 self.num_sidebar_pages = 0 self.first_chapter_page = -1 self.first_link_page = -1 - self.first_bkinfo_page = -1 + self.first_bookinfo_page = -1 self.first_footnote_page = -1 self.first_xtextsize_page = -1 self.first_sidebar_page = -1 - if DEBUG: - print "num_text_pages: %d" % self.num_text_pages - print "first_text_page: %d" % 1 - - print "num_chapter_pages: %d" % self.num_chapter_pages - print "first_chapter_page: %d" % self.first_chapter_page - - print "num_image_pages: %d" % self.num_image_pages - print "first_image_page: %d" % self.first_image_page - - print "num_footnote_pages: %d" % self.num_footnote_pages - print "first_footnote_page: %d" % self.first_footnote_page - - print "num_link_pages: %d" % self.num_link_pages - print "first_link_page: %d" % self.first_link_page - - print "num_bkinfo_pages: %d" % self.num_bkinfo_pages - print "first_bkinfo_page: %d" % self.first_bkinfo_page - - print "num_xtextsize_pages: %d" % self.num_xtextsize_pages - print "first_xtextsize_page: %d" % self.first_xtextsize_page - - print "num_sidebar_pages: %d" % self.num_sidebar_pages - print "first_sidebar_page: %d" % self.first_sidebar_page - logging.debug('self.num_text_pages %d', self.num_text_pages) logging.debug('self.num_footnote_pages %d, self.first_footnote_page %d', self.num_footnote_pages , self.first_footnote_page) - + logging.debug('self.num_sidebar_pages %d, self.first_sidebar_page %d', self.num_sidebar_pages , self.first_sidebar_page) self.flags = struct.unpack('>L', r[4:8])[0] reqd_flags = (1<<9) | (1<<7) | (1<<10) if (self.flags & reqd_flags) != reqd_flags: print "Flags: 0x%X" % self.flags raise ValueError('incompatible eReader file') - # the user_key is used to unpack the encrypted key which is stored in the unshuffled data des = Des(fixKey(user_key)) if version == 259: if drm_sub_version != 7: @@ -488,71 +434,61 @@ class EreaderProcessor(object): elif version == 272: encrypted_key = r[172:172+8] encrypted_key_sha = r[56:56+20] - - # the decrypted version of encrypted_key is the content_key self.content_key = des.decrypt(encrypted_key) - if sha1(self.content_key).digest() != encrypted_key_sha: raise ValueError('Incorrect Name and/or Credit Card') - + def getNumImages(self): return self.num_image_pages - + def getImage(self, i): sect = self.section_reader(self.first_image_page + i) name = sect[4:4+32].strip('\0') data = sect[62:] return sanitizeFileName(name), data - - + def getChapterNamePMLOffsetData(self): cv = '' if self.num_chapter_pages > 0: - # now dump chapter offsets and chapter names - # see mobile read wiki for details for i in xrange(self.num_chapter_pages): chaps = self.section_reader(self.first_chapter_page + i) j = i % self.xortable_size - cv += deXOR(chaps, j, self.xortable) + offname = deXOR(chaps, j, self.xortable) + offset = struct.unpack('>L', offname[0:4])[0] + name = offname[4:].strip('\0') + cv += '%d,%s\n' % (offset, name) return cv def getLinkNamePMLOffsetData(self): lv = '' if self.num_link_pages > 0: - # now dump link offset and link names - # see mobileread wiki for details for i in xrange(self.num_link_pages): links = self.section_reader(self.first_link_page + i) j = i % self.xortable_size - lv += deXOR(links, j, self.xortable) + offname = deXOR(links, j, self.xortable) + offset = struct.unpack('>L', offname[0:4])[0] + name = offname[4:].strip('\0') + lv += '%d,%s\n' % (offset, name) return lv - - def getBookInfoData(self): - bi = '' - if self.num_bkinfo_pages > 0: - # now dump book information - # see mobileread wiki for details - for i in xrange(self.num_bkinfo_pages): - binfo = self.section_reader(self.first_bkinfo_page + i) - j = i % self.xortable_size - bi += deXOR(binfo, j, self.xortable) - return bi - - def getExpandedTextSizesData(self): - ts = '' - if self.num_xtextsize_pages > 0: - # now dump table of expanded sizes for each text page - # (two bytes for each text page - see mobileread wiki for details) - for i in xrange(self.num_xtextsize_pages): - tsize = self.section_reader(self.first_xtextsize_page + i) - j = i % self.xortable_size - ts += deXOR(tsize, j, self.xortable) - return ts + ts = '' + if self.num_xtextsize_pages > 0: + tsize = deXOR(self.section_reader(self.first_xtextsize_page), 0, self.xortable) + for i in xrange(self.num_text_pages): + xsize = struct.unpack('>H', tsize[0:2])[0] + ts += "%d\n" % xsize + tsize = tsize[2:] + return ts + + def getBookInfo(self): + bkinfo = '' + if self.num_bookinfo_pages > 0: + info = self.section_reader(self.first_bookinfo_page) + bkinfo = deXOR(info, 0, self.xortable) + return bkinfo def getText(self): - # uses the content_key and zlib to decrypt and then inflate text sections des = Des(fixKey(self.content_key)) r = '' for i in xrange(self.num_text_pages): @@ -561,15 +497,15 @@ class EreaderProcessor(object): # now handle footnotes pages if self.num_footnote_pages > 0: - # the first record of the footnote section must pass through the Xor Table to make it useful + # the record 0 of the footnote section must pass through the Xor Table to make it useful sect = self.section_reader(self.first_footnote_page) fnote_ids = deXOR(sect, 0, self.xortable) - p = 0 # the remaining records of the footnote sections need to be decoded with the content_key and zlib inflated des = Des(fixKey(self.content_key)) r += '\\w="100%"' r += 'Footnotes\p' for i in xrange(1,self.num_footnote_pages): + logging.debug('get footnotepage %d', i) id_len = ord(fnote_ids[2]) id = fnote_ids[3:3+id_len] fmarker='\Q="%s"' % id @@ -578,14 +514,30 @@ class EreaderProcessor(object): r += '\n' fnote_ids = fnote_ids[id_len+4:] - # TO-DO - handle sidebar pages similar to footnotes pages - + # now handle sidebar pages + if self.num_sidebar_pages > 0: + # the record 0 of the sidebar section must pass through the Xor Table to make it useful + sect = self.section_reader(self.first_sidebar_page) + sbar_ids = deXOR(sect, 0, self.xortable) + # the remaining records of the sidebar sections need to be decoded with the content_key and zlib inflated + des = Des(fixKey(self.content_key)) + r += '\\w="100%"' + r += 'Sidebars\p' + for i in xrange(1,self.num_sidebar_pages): + id_len = ord(sbar_ids[2]) + id = sbar_ids[3:3+id_len] + fmarker='\Q="%s"' % id + r+=fmarker + r += zlib.decompress(des.decrypt(self.section_reader(self.first_sidebar_page + i))) + r += '\n' + sbar_ids = sbar_ids[id_len+4:] return r class PmlConverter(object): - def __init__(self, s): + def __init__(self, s, bkinfo): self.s = s self.pos = 0 + self.bkinfo = bkinfo def nextOptAttr(self): p = self.pos if self.s[p:p+2] != '="': @@ -630,12 +582,16 @@ class PmlConverter(object): print "unknown escape code %s" % c self.pos = p + 1 return None, None, None - def linkPrinter(link): - return '' % link - def footnotePrinter(link): - return '' % link ## TODO may need to cgi.escape name.... def LinePrinter(link): - return '
' % link + return '
' % link + def LinkPrinter(link): + return '' % link + def InternalLinkPrinter(link): + return '' % link + def NotSupported(link): + raise NotImplemented() + def NewChapterNewPage(link): + raise NotImplemented() def ChapterTitle(link): print "Nonfatal Error: ChapterTitle not implemented." return '' %link @@ -657,12 +613,11 @@ class PmlConverter(object): def ReferenceIndexItem(link): print "Nonfatal Error: ReferenceIndexItem not implemented." return '' %link - # See http://wiki.mobileread.com/wiki/PML#Palm_Markup_Language html_tags = { - 'c' : ('
', '
\n'), - 'r' : ('
', '
\n'), + 'c' : ('
', '
'), + 'r' : ('
', '
'), 'i' : ('', ''), 'u' : ('', ''), 'b' : ('', ''), @@ -672,30 +627,30 @@ class PmlConverter(object): 't' : ('', ''), 'Sb' : ('', ''), 'Sp' : ('', ''), - 'X0' : ('

', '

\n'), - 'X1' : ('

', '

\n'), - 'X2' : ('

', '

\n'), - 'X3' : ('

', '

\n'), - 'X4' : ('
', '
\n'), + 'X0' : ('

', '

'), + 'X1' : ('

', '

'), + 'X2' : ('

', '

'), + 'X3' : ('

', '

'), + 'X4' : ('
', '
'), 'l' : ('', ''), - 'q' : (linkPrinter, ''), - 'Fn' : (footnotePrinter, ''), + 'q' : (LinkPrinter, ''), + 'Fn' : (InternalLinkPrinter, ''), + 'Sd' : (InternalLinkPrinter, ''), 'w' : (LinePrinter, ''), #'m' : handled in if block, #'Q' : handled in if block, #'a' : handled in if block, #'U' : handled in if block, - 'x' : ('

', '

\n'), + 'x' : ('

class="breakbefore">', '

'), 'Cn' : (ChapterTitle, ''), 'T' : (IndentPercent, ''), 'n' : (NormalFont, ''), #'s' : (StdFont, ''), 's' : ('', ''), - 'k' : ('', ''), # NOTE some pdb's then go ahead and use uppercase letters - which doesn't format the way one would expect (perhaps post process the output with html dom and lower case if only upper case letters are found?) + 'k' : ('', ''), # NOTE some pdb's then go ahead and use uppercase letters - which doesn't format the way one would expect (perhaps post process the output with html dom and lower ase is only upper case letters are found?) '\\' : (SingleBackslash, ''), '-' : (SoftHyphen, ''), 'I' : (ReferenceIndexItem, ''), - 'Sd' : (footnotePrinter, ''), ## untested } html_one_tags = { 'p' : '

 

\n' @@ -709,7 +664,18 @@ class PmlConverter(object): 156: 'œ',159: 'Ÿ' } def process(self): - final = '\n\n\n\n\n\n\n' + final = '\n' + final += '\n\n\n' + if len(self.bkinfo) > 0: + title, author, copyright, publisher, isbn = self.bkinfo.split('\0',4) + isbn = isbn.strip('\0') + final += '\n' % title + final += '\n' % author + final += '\n' % copyright + final += '\n' % publisher + final += '\n' % isbn + final += '\n' + final += '\n' in_tags = [] def makeText(s): s = s.replace('&', '&') @@ -732,7 +698,7 @@ class PmlConverter(object): if type(r) != str: r = r(attr) return r - + if cmd in self.html_tags: pair = (cmd, attr) if cmd not in [a for (a,b) in in_tags]: @@ -749,7 +715,7 @@ class PmlConverter(object): while j < len(in_tags): final += getTag(in_tags[j], False) j = j + 1 - + if cmd in self.html_one_tags: final += self.html_one_tags[cmd] if cmd == 'm': @@ -781,29 +747,30 @@ def convertEreaderToHtml(infile, name, cc, outdir): imagedirpath = os.path.join(outdir,imagedir) if not os.path.exists(imagedirpath): os.makedirs(imagedirpath) - for i in xrange(er.getNumImages()): - name, contents = er.getImage(i) - file(os.path.join(imagedirpath, name), 'wb').write(contents) + for i in xrange(er.getNumImages()): + name, contents = er.getImage(i) + file(os.path.join(imagedirpath, name), 'wb').write(contents) pml_string = er.getText() - pml = PmlConverter(pml_string) pmlfilename = bookname + ".pml" - htmlfilename = bookname + ".html" file(os.path.join(outdir, pmlfilename),'wb').write(pml_string) + + bkinfo = er.getBookInfo() + + pml = PmlConverter(pml_string, bkinfo) + htmlfilename = bookname + ".html" file(os.path.join(outdir, htmlfilename),'wb').write(pml.process()) - + ts = er.getExpandedTextSizesData() file(os.path.join(outdir, 'xtextsizes.dat'), 'wb').write(ts) cv = er.getChapterNamePMLOffsetData() file(os.path.join(outdir, 'chapters.dat'), 'wb').write(cv) - - bi = er.getBookInfoData() - file(os.path.join(outdir, 'bookinfo.dat'), 'wb').write(bi) - + lv = er.getLinkNamePMLOffsetData() file(os.path.join(outdir, 'links.dat'), 'wb').write(lv) + def main(argv=None): global bookname if argv is None: @@ -825,7 +792,7 @@ def main(argv=None): elif len(argv)==5: infile, outdir, name, cc = argv[1], argv[2], argv[3], argv[4] bookname = os.path.splitext(os.path.basename(infile))[0] - + try: print "Processing..." import time @@ -845,4 +812,3 @@ if __name__ == "__main__": #cProfile.runctx( command, globals(), locals(), filename="cprofile.profile" ) sys.exit(main()) -