From 322bcdadf439963ee372828ea094604fabfdc9c8 Mon Sep 17 00:00:00 2001 From: Johannes 'josch' Schauer Date: Fri, 23 Nov 2018 18:18:23 +0100 Subject: import upstream version 0.3.2 --- CHANGES.rst | 11 ++ PKG-INFO | 8 +- setup.py | 2 +- src/img2pdf.egg-info/PKG-INFO | 8 +- src/img2pdf.py | 228 ++++++++++++++++++++++++++++++------------ 5 files changed, 184 insertions(+), 73 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4f5bee3..cbe43ce 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -2,6 +2,17 @@ CHANGES ======= +0.3.2 (2018-11-20) +------------------ + + - support big endian TIFF with lsb-to-msb FillOrder + - support multipage CCITT Group 4 TIFF + - also reject palette images with transparency + - support PNG images with 1, 2, 4 or 16 bits per sample + - support multipage TIFF with differently encoded images + - support CCITT Group4 TIFF without rows-per-strip + - add extensive test suite + 0.3.1 (2018-08-04) ------------------ diff --git a/PKG-INFO b/PKG-INFO index 975388d..7925752 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,13 +1,12 @@ -Metadata-Version: 1.1 +Metadata-Version: 2.1 Name: img2pdf -Version: 0.3.1 +Version: 0.3.2 Summary: Convert images to PDF via direct JPEG inclusion. Home-page: https://gitlab.mister-muffin.de/josch/img2pdf Author: Johannes 'josch' Schauer Author-email: josch@mister-muffin.de License: LGPL -Download-URL: https://gitlab.mister-muffin.de/josch/img2pdf/repository/archive.tar.gz?ref=0.3.1 -Description-Content-Type: UNKNOWN +Download-URL: https://gitlab.mister-muffin.de/josch/img2pdf/repository/archive.tar.gz?ref=0.3.2 Description: img2pdf ======= @@ -243,3 +242,4 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3) Classifier: Natural Language :: English Classifier: Operating System :: OS Independent +Provides-Extra: test diff --git a/setup.py b/setup.py index cc56301..8160035 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup PY3 = sys.version_info[0] >= 3 -VERSION = "0.3.1" +VERSION = "0.3.2" INSTALL_REQUIRES = ( 'Pillow', diff --git a/src/img2pdf.egg-info/PKG-INFO b/src/img2pdf.egg-info/PKG-INFO index 975388d..7925752 100644 --- a/src/img2pdf.egg-info/PKG-INFO +++ b/src/img2pdf.egg-info/PKG-INFO @@ -1,13 +1,12 @@ -Metadata-Version: 1.1 +Metadata-Version: 2.1 Name: img2pdf -Version: 0.3.1 +Version: 0.3.2 Summary: Convert images to PDF via direct JPEG inclusion. Home-page: https://gitlab.mister-muffin.de/josch/img2pdf Author: Johannes 'josch' Schauer Author-email: josch@mister-muffin.de License: LGPL -Download-URL: https://gitlab.mister-muffin.de/josch/img2pdf/repository/archive.tar.gz?ref=0.3.1 -Description-Content-Type: UNKNOWN +Download-URL: https://gitlab.mister-muffin.de/josch/img2pdf/repository/archive.tar.gz?ref=0.3.2 Description: img2pdf ======= @@ -243,3 +242,4 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3) Classifier: Natural Language :: English Classifier: Operating System :: OS Independent +Provides-Extra: test diff --git a/src/img2pdf.py b/src/img2pdf.py index 7c1978e..e9402b3 100755 --- a/src/img2pdf.py +++ b/src/img2pdf.py @@ -23,6 +23,7 @@ import os import zlib import argparse from PIL import Image, TiffImagePlugin +#TiffImagePlugin.DEBUG = True from datetime import datetime from jp2 import parsejp2 from enum import Enum @@ -32,7 +33,7 @@ import struct PY3 = sys.version_info[0] >= 3 -__version__ = "0.3.1" +__version__ = "0.3.2" default_dpi = 96.0 papersizes = { "letter": "8.5inx11in", @@ -77,6 +78,39 @@ Unit = Enum('Unit', 'pt cm mm inch') ImgUnit = Enum('ImgUnit', 'pt cm mm inch perc dpi') +TIFFBitRevTable = [ + 0x00, 0x80, 0x40, 0xC0, 0x20, 0xA0, 0x60, 0xE0, + 0x10, 0x90, 0x50, 0xD0, 0x30, 0xB0, 0x70, 0xF0, + 0x08, 0x88, 0x48, 0xC8, 0x28, 0xA8, 0x68, 0xE8, + 0x18, 0x98, 0x58, 0xD8, 0x38, 0xB8, 0x78, 0xF8, + 0x04, 0x84, 0x44, 0xC4, 0x24, 0xA4, 0x64, 0xE4, + 0x14, 0x94, 0x54, 0xD4, 0x34, 0xB4, 0x74, 0xF4, + 0x0C, 0x8C, 0x4C, 0xCC, 0x2C, 0xAC, 0x6C, 0xEC, + 0x1C, 0x9C, 0x5C, 0xDC, 0x3C, 0xBC, 0x7C, 0xFC, + 0x02, 0x82, 0x42, 0xC2, 0x22, 0xA2, 0x62, 0xE2, + 0x12, 0x92, 0x52, 0xD2, 0x32, 0xB2, 0x72, 0xF2, + 0x0A, 0x8A, 0x4A, 0xCA, 0x2A, 0xAA, 0x6A, 0xEA, + 0x1A, 0x9A, 0x5A, 0xDA, 0x3A, 0xBA, 0x7A, 0xFA, + 0x06, 0x86, 0x46, 0xC6, 0x26, 0xA6, 0x66, 0xE6, + 0x16, 0x96, 0x56, 0xD6, 0x36, 0xB6, 0x76, 0xF6, + 0x0E, 0x8E, 0x4E, 0xCE, 0x2E, 0xAE, 0x6E, 0xEE, + 0x1E, 0x9E, 0x5E, 0xDE, 0x3E, 0xBE, 0x7E, 0xFE, + 0x01, 0x81, 0x41, 0xC1, 0x21, 0xA1, 0x61, 0xE1, + 0x11, 0x91, 0x51, 0xD1, 0x31, 0xB1, 0x71, 0xF1, + 0x09, 0x89, 0x49, 0xC9, 0x29, 0xA9, 0x69, 0xE9, + 0x19, 0x99, 0x59, 0xD9, 0x39, 0xB9, 0x79, 0xF9, + 0x05, 0x85, 0x45, 0xC5, 0x25, 0xA5, 0x65, 0xE5, + 0x15, 0x95, 0x55, 0xD5, 0x35, 0xB5, 0x75, 0xF5, + 0x0D, 0x8D, 0x4D, 0xCD, 0x2D, 0xAD, 0x6D, 0xED, + 0x1D, 0x9D, 0x5D, 0xDD, 0x3D, 0xBD, 0x7D, 0xFD, + 0x03, 0x83, 0x43, 0xC3, 0x23, 0xA3, 0x63, 0xE3, + 0x13, 0x93, 0x53, 0xD3, 0x33, 0xB3, 0x73, 0xF3, + 0x0B, 0x8B, 0x4B, 0xCB, 0x2B, 0xAB, 0x6B, 0xEB, + 0x1B, 0x9B, 0x5B, 0xDB, 0x3B, 0xBB, 0x7B, 0xFB, + 0x07, 0x87, 0x47, 0xC7, 0x27, 0xA7, 0x67, 0xE7, + 0x17, 0x97, 0x57, 0xD7, 0x37, 0xB7, 0x77, 0xF7, + 0x0F, 0x8F, 0x4F, 0xCF, 0x2F, 0xAF, 0x6F, 0xEF, + 0x1F, 0x9F, 0x5F, 0xDF, 0x3F, 0xBF, 0x7F, 0xFF] class NegativeDimensionError(Exception): pass @@ -284,6 +318,10 @@ if PY3: string = string.encode('ascii') except UnicodeEncodeError: string = b"\xfe\xff"+string.encode("utf-16-be") + # We should probably encode more here because at least + # ghostscript interpretes a carriage return byte (0x0D) as a + # new line byte (0x0A) + # PDF supports: \n, \r, \t, \b and \f string = string.replace(b'\\', b'\\\\') string = string.replace(b'(', b'\\(') string = string.replace(b')', b'\\)') @@ -376,7 +414,8 @@ class pdfdoc(object): def add_imagepage(self, color, imgwidthpx, imgheightpx, imgformat, imgdata, imgwidthpdf, imgheightpdf, imgxpdf, imgypdf, pagewidth, - pageheight, userunit=None, palette=None, inverted=False): + pageheight, userunit=None, palette=None, inverted=False, + depth=0): if self.with_pdfrw: from pdfrw import PdfDict, PdfName, PdfObject, PdfString from pdfrw.py23_diffs import convert_load @@ -423,21 +462,7 @@ class pdfdoc(object): image[PdfName.Width] = imgwidthpx image[PdfName.Height] = imgheightpx image[PdfName.ColorSpace] = colorspace - # hardcoded as PIL doesn't provide bits for non-jpeg formats - if imgformat is ImageFormat.CCITTGroup4: - image[PdfName.BitsPerComponent] = 1 - else: - if color == Colorspace['1']: - image[PdfName.BitsPerComponent] = 1 - elif color == Colorspace.P: - if len(palette) <= 2**1: - image[PdfName.BitsPerComponent] = 1 - elif len(palette) <= 2**4: - image[PdfName.BitsPerComponent] = 4 - else: - image[PdfName.BitsPerComponent] = 8 - else: - image[PdfName.BitsPerComponent] = 8 + image[PdfName.BitsPerComponent] = depth if color == Colorspace['CMYK;I']: # Inverts all four channels @@ -463,17 +488,7 @@ class pdfdoc(object): else: decodeparms[PdfName.Colors] = 3 decodeparms[PdfName.Columns] = imgwidthpx - if color == Colorspace['1']: - decodeparms[PdfName.BitsPerComponent] = 1 - elif color == Colorspace.P: - if len(palette) <= 2**1: - decodeparms[PdfName.BitsPerComponent] = 1 - elif len(palette) <= 2**4: - decodeparms[PdfName.BitsPerComponent] = 4 - else: - decodeparms[PdfName.BitsPerComponent] = 8 - else: - decodeparms[PdfName.BitsPerComponent] = 8 + decodeparms[PdfName.BitsPerComponent] = depth image[PdfName.DecodeParms] = decodeparms text = ("q\n%0.4f 0 0 %0.4f %0.4f %0.4f cm\n/Im0 Do\nQ" % @@ -651,7 +666,7 @@ def get_imgmetadata(imgdata, imgformat, default_dpi, colorspace, rawdata=None): ndpi = (int(round(ndpi[0])), int(round(ndpi[1]))) ics = imgdata.mode - if ics in ["LA", "PA", "RGBA"]: + if ics in ["LA", "PA", "RGBA"] or "transparency" in imgdata.info: logging.warning("Image contains transparency which cannot be retained " "in PDF.") logging.warning("img2pdf will not perform a lossy operation.") @@ -667,6 +682,12 @@ def get_imgmetadata(imgdata, imgformat, default_dpi, colorspace, rawdata=None): if ndpi == (0, 0): ndpi = (default_dpi, default_dpi) + # PIL defaults to a dpi of 1 if a TIFF image does not specify the dpi. + # In that case, we want to use a different default. + if ndpi == (1, 1) and imgformat == ImageFormat.TIFF: + ndpi = (imgdata.tag_v2.get(TiffImagePlugin.X_RESOLUTION, default_dpi), + imgdata.tag_v2.get(TiffImagePlugin.Y_RESOLUTION, default_dpi)) + logging.debug("input dpi = %d x %d", *ndpi) if colorspace: @@ -678,7 +699,16 @@ def get_imgmetadata(imgdata, imgformat, default_dpi, colorspace, rawdata=None): if c.name == ics: color = c if color is None: - color = Colorspace.other + # PIL does not provide the information about the original + # colorspace for 16bit grayscale PNG images. Thus, we retrieve + # that info manually by looking at byte 10 in the IHDR chunk. We + # know where to find that in the file because the IHDR chunk must + # be the first chunk + if rawdata is not None and imgformat == ImageFormat.PNG \ + and rawdata[25] == 0: + color = Colorspace.L + else: + raise ValueError("unknown colorspace") if color == Colorspace.CMYK and imgformat == ImageFormat.JPEG: # Adobe inverts CMYK JPEGs for some reason, and others # have followed suit as well. Some software assumes the @@ -706,7 +736,7 @@ def ccitt_payload_location_from_pil(img): # Read the TIFF tags to find the offset(s) of the compressed data strips. strip_offsets = img.tag_v2[TiffImagePlugin.STRIPOFFSETS] strip_bytes = img.tag_v2[TiffImagePlugin.STRIPBYTECOUNTS] - rows_per_strip = img.tag_v2[TiffImagePlugin.ROWSPERSTRIP] + rows_per_strip = img.tag_v2.get(TiffImagePlugin.ROWSPERSTRIP, 2**32 - 1) # PIL always seems to create a single strip even for very large TIFFs when # it saves images, so assume we only have to read a single strip. @@ -717,6 +747,9 @@ def ccitt_payload_location_from_pil(img): (offset, ), (length, ) = strip_offsets, strip_bytes + logging.debug("TIFF strip_offsets: %d" % offset) + logging.debug("TIFF strip_bytes: %d" % length) + return offset, length @@ -758,6 +791,15 @@ def parse_png(rawdata): if rawdata[i-4:i] == b"IDAT": pngidat += rawdata[i:i+n] elif rawdata[i-4:i] == b"PLTE": + # This could be as simple as saying "palette = rawdata[i:i+n]" but + # pdfrw does only escape parenthesis and backslashes in the raw + # byte stream. But raw carriage return bytes are interpreted as + # line feed bytes by ghostscript. So instead we use the hex string + # format. pdfrw cannot write it but at least ghostscript is happy + # with it. We would also write out the palette in binary format + # (and escape more bytes) but since we cannot use pdfrw anyways, + # we choose the more human readable variant. + # See https://github.com/pmaupin/pdfrw/issues/147 for j in range(i, i+n, 3): # with int.from_bytes() we would not have to prepend extra # zeroes @@ -805,8 +847,9 @@ def read_images(rawdata, colorspace, first_frame_only=False): if color == Colorspace['RGBA']: raise JpegColorspaceError("jpeg can't have an alpha channel") im.close() + logging.debug("read_images() embeds a JPEG") return [(color, ndpi, imgformat, rawdata, imgwidthpx, imgheightpx, [], - False)] + False, 8)] # We can directly embed the IDAT chunk of PNG images if the PNG is not # interlaced @@ -820,31 +863,28 @@ def read_images(rawdata, colorspace, first_frame_only=False): imgdata, imgformat, default_dpi, colorspace, rawdata) pngidat, palette = parse_png(rawdata) im.close() + # PIL does not provide the information about the original bits per + # sample. Thus, we retrieve that info manually by looking at byte 9 in + # the IHDR chunk. We know where to find that in the file because the + # IHDR chunk must be the first chunk + depth = rawdata[24] + if depth not in [1, 2, 4, 8, 16]: + raise ValueError("invalid bit depth: %d" % depth) + logging.debug("read_images() embeds a PNG") return [(color, ndpi, imgformat, pngidat, imgwidthpx, imgheightpx, - palette, False)] - - # We can directly copy the data out of a CCITT Group 4 encoded TIFF, if it - # only contains a single strip - if imgformat == ImageFormat.TIFF \ - and imgdata.info['compression'] == "group4" \ - and len(imgdata.tag_v2[TiffImagePlugin.STRIPOFFSETS]) == 1: - photo = imgdata.tag_v2[TiffImagePlugin.PHOTOMETRIC_INTERPRETATION] - inverted = False - if photo == 0: - inverted = True - elif photo != 1: - raise ValueError("unsupported photometric interpretation for " - "group4 tiff: %d" % photo) - color, ndpi, imgwidthpx, imgheightpx = get_imgmetadata( - imgdata, imgformat, default_dpi, colorspace, rawdata) - offset, length = ccitt_payload_location_from_pil(imgdata) - im.seek(offset) - rawdata = im.read(length) - im.close() - return [(color, ndpi, ImageFormat.CCITTGroup4, rawdata, imgwidthpx, - imgheightpx, [], inverted)] + palette, False, depth)] - # Everything else has to be encoded + # If our input is not JPEG or PNG, then we might have a format that + # supports multiple frames (like TIFF or GIF), so we need a loop to + # iterate through all frames of the image. + # + # Each frame gets compressed using PNG compression *except* if: + # + # * The image is monochrome => encode using CCITT group 4 + # + # * The image is CMYK => zip plain RGB data + # + # * We are handling a CCITT encoded TIFF frame => embed data result = [] img_page_count = 0 @@ -858,6 +898,56 @@ def read_images(rawdata, colorspace, first_frame_only=False): if first_frame_only and img_page_count > 0: break + # PIL is unable to preserve the data of 16-bit RGB TIFF files and will + # convert it to 8-bit without the possibility to retrieve the original + # data + # https://github.com/python-pillow/Pillow/issues/1888 + # + # Some tiff images do not have BITSPERSAMPLE set. Use this to create + # such a tiff: tiffset -u 258 test.tif + if imgformat == ImageFormat.TIFF \ + and max(imgdata.tag_v2.get(TiffImagePlugin.BITSPERSAMPLE, [1])) > 8: + raise ValueError("PIL is unable to preserve more than 8 bits per sample") + + # We can directly copy the data out of a CCITT Group 4 encoded TIFF, if it + # only contains a single strip + if imgformat == ImageFormat.TIFF \ + and imgdata.info['compression'] == "group4" \ + and len(imgdata.tag_v2[TiffImagePlugin.STRIPOFFSETS]) == 1: + photo = imgdata.tag_v2[TiffImagePlugin.PHOTOMETRIC_INTERPRETATION] + inverted = False + if photo == 0: + inverted = True + elif photo != 1: + raise ValueError("unsupported photometric interpretation for " + "group4 tiff: %d" % photo) + color, ndpi, imgwidthpx, imgheightpx = get_imgmetadata( + imgdata, imgformat, default_dpi, colorspace, rawdata) + offset, length = ccitt_payload_location_from_pil(imgdata) + im.seek(offset) + rawdata = im.read(length) + fillorder = imgdata.tag_v2.get(TiffImagePlugin.FILLORDER) + if fillorder is None: + # no FillOrder: nothing to do + pass + elif fillorder == 1: + # msb-to-lsb: nothing to do + pass + elif fillorder == 2: + logging.debug("fillorder is lsb-to-msb => reverse bits") + # lsb-to-msb: reverse bits of each byte + rawdata = bytearray(rawdata) + for i in range(len(rawdata)): + rawdata[i] = TIFFBitRevTable[rawdata[i]] + rawdata = bytes(rawdata) + else: + raise ValueError("unsupported FillOrder: %d" % fillorder) + logging.debug("read_images() embeds Group4 from TIFF") + result.append((color, ndpi, ImageFormat.CCITTGroup4, rawdata, + imgwidthpx, imgheightpx, [], inverted, 1)) + img_page_count += 1 + continue + logging.debug("Converting frame: %d" % img_page_count) color, ndpi, imgwidthpx, imgheightpx = get_imgmetadata( @@ -867,9 +957,10 @@ def read_images(rawdata, colorspace, first_frame_only=False): if color == Colorspace['1']: try: ccittdata = transcode_monochrome(imgdata) - imgformat = ImageFormat.CCITTGroup4 - result.append((color, ndpi, imgformat, ccittdata, - imgwidthpx, imgheightpx, [], False)) + logging.debug( + "read_images() encoded a B/W image as CCITT group 4") + result.append((color, ndpi, ImageFormat.CCITTGroup4, ccittdata, + imgwidthpx, imgheightpx, [], False, 1)) img_page_count += 1 continue except Exception as e: @@ -888,8 +979,9 @@ def read_images(rawdata, colorspace, first_frame_only=False): # compression if color in [Colorspace.CMYK, Colorspace["CMYK;I"]]: imggz = zlib.compress(newimg.tobytes()) + logging.debug("read_images() encoded CMYK with flate compression") result.append((color, ndpi, imgformat, imggz, imgwidthpx, - imgheightpx, [], False)) + imgheightpx, [], False, 8)) else: # cheapo version to retrieve a PNG encoding of the payload is to # just save it with PIL. In the future this could be replaced by @@ -897,9 +989,17 @@ def read_images(rawdata, colorspace, first_frame_only=False): pngbuffer = BytesIO() newimg.save(pngbuffer, format="png") pngidat, palette = parse_png(pngbuffer.getvalue()) - imgformat = ImageFormat.PNG - result.append((color, ndpi, imgformat, pngidat, imgwidthpx, - imgheightpx, palette, False)) + # PIL does not provide the information about the original bits per + # sample. Thus, we retrieve that info manually by looking at byte 9 in + # the IHDR chunk. We know where to find that in the file because the + # IHDR chunk must be the first chunk + pngbuffer.seek(24) + depth = ord(pngbuffer.read(1)) + if depth not in [1, 2, 4, 8, 16]: + raise ValueError("invalid bit depth: %d" % depth) + logging.debug("read_images() encoded an image as PNG") + result.append((color, ndpi, ImageFormat.PNG, pngidat, imgwidthpx, + imgheightpx, palette, False, depth)) img_page_count += 1 # the python-pil version 2.3.0-1ubuntu3 in Ubuntu does not have the # close() method @@ -1215,7 +1315,7 @@ def convert(*images, **kwargs): rawdata = img for color, ndpi, imgformat, imgdata, imgwidthpx, imgheightpx, \ - palette, inverted in read_images( + palette, inverted, depth in read_images( rawdata, kwargs['colorspace'], kwargs['first_frame_only']): pagewidth, pageheight, imgwidthpdf, imgheightpdf = \ kwargs['layout_fun'](imgwidthpx, imgheightpx, ndpi) @@ -1240,7 +1340,7 @@ def convert(*images, **kwargs): pdf.add_imagepage(color, imgwidthpx, imgheightpx, imgformat, imgdata, imgwidthpdf, imgheightpdf, imgxpdf, imgypdf, pagewidth, pageheight, userunit, - palette, inverted) + palette, inverted, depth) if kwargs['outputstream']: pdf.tostream(kwargs['outputstream']) -- cgit v1.2.3