import upstream version 0.3.2

5 files changed, 184 insertions, 73 deletions

diff --git a/CHANGES.rst b/CHANGES.rst
index 4f5bee3..cbe43ce 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -2,6 +2,17 @@
 CHANGES
 =======
 
+0.3.2 (2018-11-20)
+------------------
+
+ - support big endian TIFF with lsb-to-msb FillOrder
+ - support multipage CCITT Group 4 TIFF
+ - also reject palette images with transparency
+ - support PNG images with 1, 2, 4 or 16 bits per sample
+ - support multipage TIFF with differently encoded images
+ - support CCITT Group4 TIFF without rows-per-strip
+ - add extensive test suite
+
 0.3.1 (2018-08-04)
 ------------------
 
diff --git a/PKG-INFO b/PKG-INFO
index 975388d..7925752 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,13 +1,12 @@
-Metadata-Version: 1.1
+Metadata-Version: 2.1
 Name: img2pdf
-Version: 0.3.1
+Version: 0.3.2
 Summary: Convert images to PDF via direct JPEG inclusion.
 Home-page: https://gitlab.mister-muffin.de/josch/img2pdf
 Author: Johannes 'josch' Schauer
 Author-email: josch@mister-muffin.de
 License: LGPL
-Download-URL: https://gitlab.mister-muffin.de/josch/img2pdf/repository/archive.tar.gz?ref=0.3.1
-Description-Content-Type: UNKNOWN
+Download-URL: https://gitlab.mister-muffin.de/josch/img2pdf/repository/archive.tar.gz?ref=0.3.2
 Description: img2pdf
         =======
         
@@ -243,3 +242,4 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)
 Classifier: Natural Language :: English
 Classifier: Operating System :: OS Independent
+Provides-Extra: test
diff --git a/setup.py b/setup.py
index cc56301..8160035 100644
--- a/setup.py
+++ b/setup.py
@@ -3,7 +3,7 @@ from setuptools import setup
 
 PY3 = sys.version_info[0] >= 3
 
-VERSION = "0.3.1"
+VERSION = "0.3.2"
 
 INSTALL_REQUIRES = (
     'Pillow',
diff --git a/src/img2pdf.egg-info/PKG-INFO b/src/img2pdf.egg-info/PKG-INFO
index 975388d..7925752 100644
--- a/src/img2pdf.egg-info/PKG-INFO
+++ b/src/img2pdf.egg-info/PKG-INFO
@@ -1,13 +1,12 @@
-Metadata-Version: 1.1
+Metadata-Version: 2.1
 Name: img2pdf
-Version: 0.3.1
+Version: 0.3.2
 Summary: Convert images to PDF via direct JPEG inclusion.
 Home-page: https://gitlab.mister-muffin.de/josch/img2pdf
 Author: Johannes 'josch' Schauer
 Author-email: josch@mister-muffin.de
 License: LGPL
-Download-URL: https://gitlab.mister-muffin.de/josch/img2pdf/repository/archive.tar.gz?ref=0.3.1
-Description-Content-Type: UNKNOWN
+Download-URL: https://gitlab.mister-muffin.de/josch/img2pdf/repository/archive.tar.gz?ref=0.3.2
 Description: img2pdf
         =======
         
@@ -243,3 +242,4 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
 Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)
 Classifier: Natural Language :: English
 Classifier: Operating System :: OS Independent
+Provides-Extra: test
diff --git a/src/img2pdf.py b/src/img2pdf.py
index 7c1978e..e9402b3 100755
--- a/src/img2pdf.py
+++ b/src/img2pdf.py
@@ -23,6 +23,7 @@ import os
 import zlib
 import argparse
 from PIL import Image, TiffImagePlugin
+#TiffImagePlugin.DEBUG = True
 from datetime import datetime
 from jp2 import parsejp2
 from enum import Enum
@@ -32,7 +33,7 @@ import struct
 
 PY3 = sys.version_info[0] >= 3
 
-__version__ = "0.3.1"
+__version__ = "0.3.2"
 default_dpi = 96.0
 papersizes = {
     "letter": "8.5inx11in",
@@ -77,6 +78,39 @@ Unit = Enum('Unit', 'pt cm mm inch')
 
 ImgUnit = Enum('ImgUnit', 'pt cm mm inch perc dpi')
 
+TIFFBitRevTable = [
+        0x00, 0x80, 0x40, 0xC0, 0x20, 0xA0, 0x60, 0xE0,
+        0x10, 0x90, 0x50, 0xD0, 0x30, 0xB0, 0x70, 0xF0,
+        0x08, 0x88, 0x48, 0xC8, 0x28, 0xA8, 0x68, 0xE8,
+        0x18, 0x98, 0x58, 0xD8, 0x38, 0xB8, 0x78, 0xF8,
+        0x04, 0x84, 0x44, 0xC4, 0x24, 0xA4, 0x64, 0xE4,
+        0x14, 0x94, 0x54, 0xD4, 0x34, 0xB4, 0x74, 0xF4,
+        0x0C, 0x8C, 0x4C, 0xCC, 0x2C, 0xAC, 0x6C, 0xEC,
+        0x1C, 0x9C, 0x5C, 0xDC, 0x3C, 0xBC, 0x7C, 0xFC,
+        0x02, 0x82, 0x42, 0xC2, 0x22, 0xA2, 0x62, 0xE2,
+        0x12, 0x92, 0x52, 0xD2, 0x32, 0xB2, 0x72, 0xF2,
+        0x0A, 0x8A, 0x4A, 0xCA, 0x2A, 0xAA, 0x6A, 0xEA,
+        0x1A, 0x9A, 0x5A, 0xDA, 0x3A, 0xBA, 0x7A, 0xFA,
+        0x06, 0x86, 0x46, 0xC6, 0x26, 0xA6, 0x66, 0xE6,
+        0x16, 0x96, 0x56, 0xD6, 0x36, 0xB6, 0x76, 0xF6,
+        0x0E, 0x8E, 0x4E, 0xCE, 0x2E, 0xAE, 0x6E, 0xEE,
+        0x1E, 0x9E, 0x5E, 0xDE, 0x3E, 0xBE, 0x7E, 0xFE,
+        0x01, 0x81, 0x41, 0xC1, 0x21, 0xA1, 0x61, 0xE1,
+        0x11, 0x91, 0x51, 0xD1, 0x31, 0xB1, 0x71, 0xF1,
+        0x09, 0x89, 0x49, 0xC9, 0x29, 0xA9, 0x69, 0xE9,
+        0x19, 0x99, 0x59, 0xD9, 0x39, 0xB9, 0x79, 0xF9,
+        0x05, 0x85, 0x45, 0xC5, 0x25, 0xA5, 0x65, 0xE5,
+        0x15, 0x95, 0x55, 0xD5, 0x35, 0xB5, 0x75, 0xF5,
+        0x0D, 0x8D, 0x4D, 0xCD, 0x2D, 0xAD, 0x6D, 0xED,
+        0x1D, 0x9D, 0x5D, 0xDD, 0x3D, 0xBD, 0x7D, 0xFD,
+        0x03, 0x83, 0x43, 0xC3, 0x23, 0xA3, 0x63, 0xE3,
+        0x13, 0x93, 0x53, 0xD3, 0x33, 0xB3, 0x73, 0xF3,
+        0x0B, 0x8B, 0x4B, 0xCB, 0x2B, 0xAB, 0x6B, 0xEB,
+        0x1B, 0x9B, 0x5B, 0xDB, 0x3B, 0xBB, 0x7B, 0xFB,
+        0x07, 0x87, 0x47, 0xC7, 0x27, 0xA7, 0x67, 0xE7,
+        0x17, 0x97, 0x57, 0xD7, 0x37, 0xB7, 0x77, 0xF7,
+        0x0F, 0x8F, 0x4F, 0xCF, 0x2F, 0xAF, 0x6F, 0xEF,
+        0x1F, 0x9F, 0x5F, 0xDF, 0x3F, 0xBF, 0x7F, 0xFF]
 
 class NegativeDimensionError(Exception):
     pass
@@ -284,6 +318,10 @@ if PY3:
                     string = string.encode('ascii')
                 except UnicodeEncodeError:
                     string = b"\xfe\xff"+string.encode("utf-16-be")
+                # We should probably encode more here because at least
+                # ghostscript interpretes a carriage return byte (0x0D) as a
+                # new line byte (0x0A)
+                # PDF supports: \n, \r, \t, \b and \f
                 string = string.replace(b'\\', b'\\\\')
                 string = string.replace(b'(', b'\\(')
                 string = string.replace(b')', b'\\)')
@@ -376,7 +414,8 @@ class pdfdoc(object):
 
     def add_imagepage(self, color, imgwidthpx, imgheightpx, imgformat, imgdata,
                       imgwidthpdf, imgheightpdf, imgxpdf, imgypdf, pagewidth,
-                      pageheight, userunit=None, palette=None, inverted=False):
+                      pageheight, userunit=None, palette=None, inverted=False,
+                      depth=0):
         if self.with_pdfrw:
             from pdfrw import PdfDict, PdfName, PdfObject, PdfString
             from pdfrw.py23_diffs import convert_load
@@ -423,21 +462,7 @@ class pdfdoc(object):
         image[PdfName.Width] = imgwidthpx
         image[PdfName.Height] = imgheightpx
         image[PdfName.ColorSpace] = colorspace
-        # hardcoded as PIL doesn't provide bits for non-jpeg formats
-        if imgformat is ImageFormat.CCITTGroup4:
-            image[PdfName.BitsPerComponent] = 1
-        else:
-            if color == Colorspace['1']:
-                image[PdfName.BitsPerComponent] = 1
-            elif color == Colorspace.P:
-                if len(palette) <= 2**1:
-                    image[PdfName.BitsPerComponent] = 1
-                elif len(palette) <= 2**4:
-                    image[PdfName.BitsPerComponent] = 4
-                else:
-                    image[PdfName.BitsPerComponent] = 8
-            else:
-                image[PdfName.BitsPerComponent] = 8
+        image[PdfName.BitsPerComponent] = depth
 
         if color == Colorspace['CMYK;I']:
             # Inverts all four channels
@@ -463,17 +488,7 @@ class pdfdoc(object):
             else:
                 decodeparms[PdfName.Colors] = 3
             decodeparms[PdfName.Columns] = imgwidthpx
-            if color == Colorspace['1']:
-                decodeparms[PdfName.BitsPerComponent] = 1
-            elif color == Colorspace.P:
-                if len(palette) <= 2**1:
-                    decodeparms[PdfName.BitsPerComponent] = 1
-                elif len(palette) <= 2**4:
-                    decodeparms[PdfName.BitsPerComponent] = 4
-                else:
-                    decodeparms[PdfName.BitsPerComponent] = 8
-            else:
-                decodeparms[PdfName.BitsPerComponent] = 8
+            decodeparms[PdfName.BitsPerComponent] = depth
             image[PdfName.DecodeParms] = decodeparms
 
         text = ("q\n%0.4f 0 0 %0.4f %0.4f %0.4f cm\n/Im0 Do\nQ" %
@@ -651,7 +666,7 @@ def get_imgmetadata(imgdata, imgformat, default_dpi, colorspace, rawdata=None):
         ndpi = (int(round(ndpi[0])), int(round(ndpi[1])))
         ics = imgdata.mode
 
-    if ics in ["LA", "PA", "RGBA"]:
+    if ics in ["LA", "PA", "RGBA"] or "transparency" in imgdata.info:
         logging.warning("Image contains transparency which cannot be retained "
                         "in PDF.")
         logging.warning("img2pdf will not perform a lossy operation.")
@@ -667,6 +682,12 @@ def get_imgmetadata(imgdata, imgformat, default_dpi, colorspace, rawdata=None):
     if ndpi == (0, 0):
         ndpi = (default_dpi, default_dpi)
 
+    # PIL defaults to a dpi of 1 if a TIFF image does not specify the dpi.
+    # In that case, we want to use a different default.
+    if ndpi == (1, 1) and imgformat == ImageFormat.TIFF:
+        ndpi = (imgdata.tag_v2.get(TiffImagePlugin.X_RESOLUTION, default_dpi),
+                imgdata.tag_v2.get(TiffImagePlugin.Y_RESOLUTION, default_dpi))
+
     logging.debug("input dpi = %d x %d", *ndpi)
 
     if colorspace:
@@ -678,7 +699,16 @@ def get_imgmetadata(imgdata, imgformat, default_dpi, colorspace, rawdata=None):
             if c.name == ics:
                 color = c
         if color is None:
-            color = Colorspace.other
+            # PIL does not provide the information about the original
+            # colorspace for 16bit grayscale PNG images. Thus, we retrieve
+            # that info manually by looking at byte 10 in the IHDR chunk. We
+            # know where to find that in the file because the IHDR chunk must
+            # be the first chunk
+            if rawdata is not None and imgformat == ImageFormat.PNG \
+                    and rawdata[25] == 0:
+                color = Colorspace.L
+            else:
+                raise ValueError("unknown colorspace")
         if color == Colorspace.CMYK and imgformat == ImageFormat.JPEG:
             # Adobe inverts CMYK JPEGs for some reason, and others
             # have followed suit as well. Some software assumes the
@@ -706,7 +736,7 @@ def ccitt_payload_location_from_pil(img):
     # Read the TIFF tags to find the offset(s) of the compressed data strips.
     strip_offsets = img.tag_v2[TiffImagePlugin.STRIPOFFSETS]
     strip_bytes = img.tag_v2[TiffImagePlugin.STRIPBYTECOUNTS]
-    rows_per_strip = img.tag_v2[TiffImagePlugin.ROWSPERSTRIP]
+    rows_per_strip = img.tag_v2.get(TiffImagePlugin.ROWSPERSTRIP, 2**32 - 1)
 
     # PIL always seems to create a single strip even for very large TIFFs when
     # it saves images, so assume we only have to read a single strip.
@@ -717,6 +747,9 @@ def ccitt_payload_location_from_pil(img):
 
     (offset, ), (length, ) = strip_offsets, strip_bytes
 
+    logging.debug("TIFF strip_offsets: %d" % offset)
+    logging.debug("TIFF strip_bytes: %d" % length)
+
     return offset, length
 
 
@@ -758,6 +791,15 @@ def parse_png(rawdata):
         if rawdata[i-4:i] == b"IDAT":
             pngidat += rawdata[i:i+n]
         elif rawdata[i-4:i] == b"PLTE":
+            # This could be as simple as saying "palette = rawdata[i:i+n]" but
+            # pdfrw does only escape parenthesis and backslashes in the raw
+            # byte stream. But raw carriage return bytes are interpreted as
+            # line feed bytes by ghostscript. So instead we use the hex string
+            # format. pdfrw cannot write it but at least ghostscript is happy
+            # with it. We would also write out the palette in binary format
+            # (and escape more bytes) but since we cannot use pdfrw anyways,
+            # we choose the more human readable variant.
+            # See https://github.com/pmaupin/pdfrw/issues/147
             for j in range(i, i+n, 3):
                 # with int.from_bytes() we would not have to prepend extra
                 # zeroes
@@ -805,8 +847,9 @@ def read_images(rawdata, colorspace, first_frame_only=False):
         if color == Colorspace['RGBA']:
             raise JpegColorspaceError("jpeg can't have an alpha channel")
         im.close()
+        logging.debug("read_images() embeds a JPEG")
         return [(color, ndpi, imgformat, rawdata, imgwidthpx, imgheightpx, [],
-                 False)]
+                 False, 8)]
 
     # We can directly embed the IDAT chunk of PNG images if the PNG is not
     # interlaced
@@ -820,31 +863,28 @@ def read_images(rawdata, colorspace, first_frame_only=False):
                 imgdata, imgformat, default_dpi, colorspace, rawdata)
         pngidat, palette = parse_png(rawdata)
         im.close()
+        # PIL does not provide the information about the original bits per
+        # sample. Thus, we retrieve that info manually by looking at byte 9 in
+        # the IHDR chunk. We know where to find that in the file because the
+        # IHDR chunk must be the first chunk
+        depth = rawdata[24]
+        if depth not in [1, 2, 4, 8, 16]:
+            raise ValueError("invalid bit depth: %d" % depth)
+        logging.debug("read_images() embeds a PNG")
         return [(color, ndpi, imgformat, pngidat, imgwidthpx, imgheightpx,
-                 palette, False)]
-
-    # We can directly copy the data out of a CCITT Group 4 encoded TIFF, if it
-    # only contains a single strip
-    if imgformat == ImageFormat.TIFF \
-            and imgdata.info['compression'] == "group4" \
-            and len(imgdata.tag_v2[TiffImagePlugin.STRIPOFFSETS]) == 1:
-        photo = imgdata.tag_v2[TiffImagePlugin.PHOTOMETRIC_INTERPRETATION]
-        inverted = False
-        if photo == 0:
-            inverted = True
-        elif photo != 1:
-            raise ValueError("unsupported photometric interpretation for "
-                             "group4 tiff: %d" % photo)
-        color, ndpi, imgwidthpx, imgheightpx = get_imgmetadata(
-                imgdata, imgformat, default_dpi, colorspace, rawdata)
-        offset, length = ccitt_payload_location_from_pil(imgdata)
-        im.seek(offset)
-        rawdata = im.read(length)
-        im.close()
-        return [(color, ndpi, ImageFormat.CCITTGroup4, rawdata, imgwidthpx,
-                 imgheightpx, [], inverted)]
+                 palette, False, depth)]
 
-    # Everything else has to be encoded
+    # If our input is not JPEG or PNG, then we might have a format that
+    # supports multiple frames (like TIFF or GIF), so we need a loop to
+    # iterate through all frames of the image.
+    #
+    # Each frame gets compressed using PNG compression *except* if:
+    #
+    #  * The image is monochrome => encode using CCITT group 4
+    #
+    #  * The image is CMYK => zip plain RGB data
+    #
+    #  * We are handling a CCITT encoded TIFF frame => embed data
 
     result = []
     img_page_count = 0
@@ -858,6 +898,56 @@ def read_images(rawdata, colorspace, first_frame_only=False):
         if first_frame_only and img_page_count > 0:
             break
 
+        # PIL is unable to preserve the data of 16-bit RGB TIFF files and will
+        # convert it to 8-bit without the possibility to retrieve the original
+        # data
+        # https://github.com/python-pillow/Pillow/issues/1888
+        #
+        # Some tiff images do not have BITSPERSAMPLE set. Use this to create
+        # such a tiff: tiffset -u 258 test.tif
+        if imgformat == ImageFormat.TIFF \
+                and max(imgdata.tag_v2.get(TiffImagePlugin.BITSPERSAMPLE, [1])) > 8:
+            raise ValueError("PIL is unable to preserve more than 8 bits per sample")
+
+        # We can directly copy the data out of a CCITT Group 4 encoded TIFF, if it
+        # only contains a single strip
+        if imgformat == ImageFormat.TIFF \
+                and imgdata.info['compression'] == "group4" \
+                and len(imgdata.tag_v2[TiffImagePlugin.STRIPOFFSETS]) == 1:
+            photo = imgdata.tag_v2[TiffImagePlugin.PHOTOMETRIC_INTERPRETATION]
+            inverted = False
+            if photo == 0:
+                inverted = True
+            elif photo != 1:
+                raise ValueError("unsupported photometric interpretation for "
+                                 "group4 tiff: %d" % photo)
+            color, ndpi, imgwidthpx, imgheightpx = get_imgmetadata(
+                    imgdata, imgformat, default_dpi, colorspace, rawdata)
+            offset, length = ccitt_payload_location_from_pil(imgdata)
+            im.seek(offset)
+            rawdata = im.read(length)
+            fillorder = imgdata.tag_v2.get(TiffImagePlugin.FILLORDER)
+            if fillorder is None:
+                # no FillOrder: nothing to do
+                pass
+            elif fillorder == 1:
+                # msb-to-lsb: nothing to do
+                pass
+            elif fillorder == 2:
+                logging.debug("fillorder is lsb-to-msb => reverse bits")
+                # lsb-to-msb: reverse bits of each byte
+                rawdata = bytearray(rawdata)
+                for i in range(len(rawdata)):
+                    rawdata[i] = TIFFBitRevTable[rawdata[i]]
+                rawdata = bytes(rawdata)
+            else:
+                raise ValueError("unsupported FillOrder: %d" % fillorder)
+            logging.debug("read_images() embeds Group4 from TIFF")
+            result.append((color, ndpi, ImageFormat.CCITTGroup4, rawdata,
+                           imgwidthpx, imgheightpx, [], inverted, 1))
+            img_page_count += 1
+            continue
+
         logging.debug("Converting frame: %d" % img_page_count)
 
         color, ndpi, imgwidthpx, imgheightpx = get_imgmetadata(
@@ -867,9 +957,10 @@ def read_images(rawdata, colorspace, first_frame_only=False):
         if color == Colorspace['1']:
             try:
                 ccittdata = transcode_monochrome(imgdata)
-                imgformat = ImageFormat.CCITTGroup4
-                result.append((color, ndpi, imgformat, ccittdata,
-                               imgwidthpx, imgheightpx, [], False))
+                logging.debug(
+                        "read_images() encoded a B/W image as CCITT group 4")
+                result.append((color, ndpi, ImageFormat.CCITTGroup4, ccittdata,
+                               imgwidthpx, imgheightpx, [], False, 1))
                 img_page_count += 1
                 continue
             except Exception as e:
@@ -888,8 +979,9 @@ def read_images(rawdata, colorspace, first_frame_only=False):
         # compression
         if color in [Colorspace.CMYK, Colorspace["CMYK;I"]]:
             imggz = zlib.compress(newimg.tobytes())
+            logging.debug("read_images() encoded CMYK with flate compression")
             result.append((color, ndpi, imgformat, imggz, imgwidthpx,
-                           imgheightpx, [], False))
+                           imgheightpx, [], False, 8))
         else:
             # cheapo version to retrieve a PNG encoding of the payload is to
             # just save it with PIL. In the future this could be replaced by
@@ -897,9 +989,17 @@ def read_images(rawdata, colorspace, first_frame_only=False):
             pngbuffer = BytesIO()
             newimg.save(pngbuffer, format="png")
             pngidat, palette = parse_png(pngbuffer.getvalue())
-            imgformat = ImageFormat.PNG
-            result.append((color, ndpi, imgformat, pngidat, imgwidthpx,
-                           imgheightpx, palette, False))
+            # PIL does not provide the information about the original bits per
+            # sample. Thus, we retrieve that info manually by looking at byte 9 in
+            # the IHDR chunk. We know where to find that in the file because the
+            # IHDR chunk must be the first chunk
+            pngbuffer.seek(24)
+            depth = ord(pngbuffer.read(1))
+            if depth not in [1, 2, 4, 8, 16]:
+                raise ValueError("invalid bit depth: %d" % depth)
+            logging.debug("read_images() encoded an image as PNG")
+            result.append((color, ndpi, ImageFormat.PNG, pngidat, imgwidthpx,
+                           imgheightpx, palette, False, depth))
         img_page_count += 1
     # the python-pil version 2.3.0-1ubuntu3 in Ubuntu does not have the
     # close() method
@@ -1215,7 +1315,7 @@ def convert(*images, **kwargs):
                 rawdata = img
 
         for color, ndpi, imgformat, imgdata, imgwidthpx, imgheightpx, \
-                palette, inverted in read_images(
+                palette, inverted, depth in read_images(
                     rawdata, kwargs['colorspace'], kwargs['first_frame_only']):
             pagewidth, pageheight, imgwidthpdf, imgheightpdf = \
                 kwargs['layout_fun'](imgwidthpx, imgheightpx, ndpi)
@@ -1240,7 +1340,7 @@ def convert(*images, **kwargs):
             pdf.add_imagepage(color, imgwidthpx, imgheightpx, imgformat,
                               imgdata, imgwidthpdf, imgheightpdf, imgxpdf,
                               imgypdf, pagewidth, pageheight, userunit,
-                              palette, inverted)
+                              palette, inverted, depth)
 
     if kwargs['outputstream']:
         pdf.tostream(kwargs['outputstream'])