diff options
author | Colin Watson <cjwatson@debian.org> | 2019-12-09 10:07:47 +0000 |
---|---|---|
committer | Colin Watson <cjwatson@debian.org> | 2019-12-09 10:07:47 +0000 |
commit | 1ff0be431951ff9b61e39d2bb2ca9a243cb90d78 (patch) | |
tree | ae0769c6d1099bba47fef0158919dc2f7319bcab | |
parent | 1b170095f3e7d03666d8e672d3ad62355a5240dc (diff) |
New upstream version 1.6.0
-rw-r--r-- | .bumpversion.cfg | 2 | ||||
-rw-r--r-- | AUTHORS.rst | 1 | ||||
-rw-r--r-- | CHANGELOG.rst | 7 | ||||
-rw-r--r-- | PKG-INFO | 103 | ||||
-rw-r--r-- | README.rst | 98 | ||||
-rw-r--r-- | docs/conf.py | 2 | ||||
-rw-r--r-- | setup.py | 2 | ||||
-rw-r--r-- | src/tblib.egg-info/PKG-INFO | 103 | ||||
-rw-r--r-- | src/tblib.egg-info/SOURCES.txt | 3 | ||||
-rw-r--r-- | src/tblib/__init__.py | 2 | ||||
-rw-r--r-- | src/tblib/pickling_support.py | 75 | ||||
-rw-r--r-- | tests/test_pickle_exception.py | 98 |
12 files changed, 472 insertions, 24 deletions
diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 98a565a..828f46b 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.5.0 +current_version = 1.6.0 commit = True tag = True diff --git a/AUTHORS.rst b/AUTHORS.rst index 9db24b3..3800ae0 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -9,3 +9,4 @@ Authors * Jon Dufresne - https://github.com/jdufresne * Elliott Sales de Andrade - https://github.com/QuLogic * Victor Stinner - https://github.com/vstinner +* Guido Imperiale - https://github.com/crusaderky diff --git a/CHANGELOG.rst b/CHANGELOG.rst index aaab9d8..5b08ce8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,13 @@ Changelog ========= +1.6.0 (2019-12-07) +~~~~~~~~~~~~~~~~~~ + +* When pickling an Exception, also pickle its traceback and the Exception chain + (``raise ... from ...``). Contributed by Guido Imperiale in + `#53 <https://github.com/ionelmc/python-tblib/issues/53>`_. + 1.5.0 (2019-10-23) ~~~~~~~~~~~~~~~~~~ @@ -1,6 +1,6 @@ Metadata-Version: 1.2 Name: tblib -Version: 1.5.0 +Version: 1.6.0 Summary: Traceback serialization library. Home-page: https://github.com/ionelmc/python-tblib Author: Ionel Cristian Mărieș @@ -15,7 +15,7 @@ Description: ======== - Traceback serialization library. + Serialization library for Exceptions and Tracebacks. * Free software: BSD license @@ -27,6 +27,8 @@ Description: ======== * Create traceback objects from strings (the ``from_string`` method). *No pickling is used*. * Serialize tracebacks to/from plain dicts (the ``from_dict`` and ``to_dict`` methods). *No pickling is used*. * Raise the tracebacks created from the aforementioned sources. + * Pickle an Exception together with its traceback and exception chain + (``raise ... from ...``) *(Python 3 only)* **Again, note that using the pickle support is completely optional. You are solely responsible for security problems should you decide to use the pickle support.** @@ -91,8 +93,8 @@ Description: ======== >>> len(s3) > 1 True - Unpickling - ~~~~~~~~~~ + Unpickling tracebacks + ~~~~~~~~~~~~~~~~~~~~~ :: @@ -154,6 +156,92 @@ Description: ======== raise Exception('fail') Exception: fail + Pickling Exceptions together with their traceback and chain (Python 3 only) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :: + + >>> try: # doctest: +SKIP + ... try: + ... 1 / 0 + ... except Exception as e: + ... raise Exception("foo") from e + ... except Exception as e: + ... s = pickle.dumps(e) + >>> raise pickle.loads(s) # doctest: +SKIP + Traceback (most recent call last): + File "<doctest README.rst[16]>", line 3, in <module> + 1 / 0 + ZeroDivisionError: division by zero + + The above exception was the direct cause of the following exception: + + Traceback (most recent call last): + File "<doctest README.rst[17]>", line 1, in <module> + raise pickle.loads(s) + File "<doctest README.rst[16]>", line 5, in <module> + raise Exception("foo") from e + Exception: foo + + BaseException subclasses defined after calling ``pickling_support.install()`` will + **not** retain their traceback and exception chain pickling. + To cover custom Exceptions, there are three options: + + 1. Use ``@pickling_support.install`` as a decorator for each custom Exception + + .. code-block:: python + + >>> from tblib import pickling_support + >>> # Declare all imports of your package's dependencies + >>> import numpy # doctest: +SKIP + + >>> pickling_support.install() # install for all modules imported so far + + >>> @pickling_support.install + ... class CustomError(Exception): + ... pass + + Eventual subclasses of ``CustomError`` will need to be decorated again. + + 2. Invoke ``pickling_support.install()`` after all modules have been imported and all + Exception subclasses have been declared + + .. code-block:: python + + >>> # Declare all imports of your package's dependencies + >>> import numpy # doctest: +SKIP + >>> from tblib import pickling_support + + >>> # Declare your own custom Exceptions + >>> class CustomError(Exception): + ... pass + + >>> # Finally, install tblib + >>> pickling_support.install() + + 3. Selectively install tblib for Exception instances just before they are pickled + + .. code-block:: python + + pickling_support.install(<Exception instance>, [Exception instance], ...) + + The above will install tblib pickling for all listed exceptions as well as any other + exceptions in their exception chains. + + For example, one could write a wrapper to be used with + `ProcessPoolExecutor <https://docs.python.org/3/library/concurrent.futures.html>`_, + `Dask.distributed <https://distributed.dask.org/>`_, or similar libraries: + + :: + + >>> from tblib import pickling_support + >>> def wrapper(func, *args, **kwargs): + ... try: + ... return func(*args, **kwargs) + ... except Exception as e: + ... pickling_support.install(e) + ... raise + What if we have a local stack, does it show correctly ? ------------------------------------------------------- @@ -581,6 +669,13 @@ Description: ======== Changelog ========= + 1.6.0 (2019-12-07) + ~~~~~~~~~~~~~~~~~~ + + * When pickling an Exception, also pickle its traceback and the Exception chain + (``raise ... from ...``). Contributed by Guido Imperiale in + `#53 <https://github.com/ionelmc/python-tblib/issues/53>`_. + 1.5.0 (2019-10-23) ~~~~~~~~~~~~~~~~~~ @@ -51,13 +51,13 @@ Overview :alt: Supported implementations :target: https://pypi.org/project/tblib -.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-tblib/v1.5.0.svg +.. |commits-since| image:: https://img.shields.io/github/commits-since/ionelmc/python-tblib/v1.6.0.svg :alt: Commits since latest release - :target: https://github.com/ionelmc/python-tblib/compare/v1.5.0...master + :target: https://github.com/ionelmc/python-tblib/compare/v1.6.0...master .. end-badges -Traceback serialization library. +Serialization library for Exceptions and Tracebacks. * Free software: BSD license @@ -69,6 +69,8 @@ It allows you to: * Create traceback objects from strings (the ``from_string`` method). *No pickling is used*. * Serialize tracebacks to/from plain dicts (the ``from_dict`` and ``to_dict`` methods). *No pickling is used*. * Raise the tracebacks created from the aforementioned sources. +* Pickle an Exception together with its traceback and exception chain + (``raise ... from ...``) *(Python 3 only)* **Again, note that using the pickle support is completely optional. You are solely responsible for security problems should you decide to use the pickle support.** @@ -133,8 +135,8 @@ those tracebacks or print them - that should cover 99% of the usecases. >>> len(s3) > 1 True -Unpickling -~~~~~~~~~~ +Unpickling tracebacks +~~~~~~~~~~~~~~~~~~~~~ :: @@ -196,6 +198,92 @@ Raising raise Exception('fail') Exception: fail +Pickling Exceptions together with their traceback and chain (Python 3 only) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:: + + >>> try: # doctest: +SKIP + ... try: + ... 1 / 0 + ... except Exception as e: + ... raise Exception("foo") from e + ... except Exception as e: + ... s = pickle.dumps(e) + >>> raise pickle.loads(s) # doctest: +SKIP + Traceback (most recent call last): + File "<doctest README.rst[16]>", line 3, in <module> + 1 / 0 + ZeroDivisionError: division by zero + + The above exception was the direct cause of the following exception: + + Traceback (most recent call last): + File "<doctest README.rst[17]>", line 1, in <module> + raise pickle.loads(s) + File "<doctest README.rst[16]>", line 5, in <module> + raise Exception("foo") from e + Exception: foo + +BaseException subclasses defined after calling ``pickling_support.install()`` will +**not** retain their traceback and exception chain pickling. +To cover custom Exceptions, there are three options: + +1. Use ``@pickling_support.install`` as a decorator for each custom Exception + + .. code-block:: python + + >>> from tblib import pickling_support + >>> # Declare all imports of your package's dependencies + >>> import numpy # doctest: +SKIP + + >>> pickling_support.install() # install for all modules imported so far + + >>> @pickling_support.install + ... class CustomError(Exception): + ... pass + + Eventual subclasses of ``CustomError`` will need to be decorated again. + +2. Invoke ``pickling_support.install()`` after all modules have been imported and all + Exception subclasses have been declared + + .. code-block:: python + + >>> # Declare all imports of your package's dependencies + >>> import numpy # doctest: +SKIP + >>> from tblib import pickling_support + + >>> # Declare your own custom Exceptions + >>> class CustomError(Exception): + ... pass + + >>> # Finally, install tblib + >>> pickling_support.install() + +3. Selectively install tblib for Exception instances just before they are pickled + + .. code-block:: python + + pickling_support.install(<Exception instance>, [Exception instance], ...) + + The above will install tblib pickling for all listed exceptions as well as any other + exceptions in their exception chains. + + For example, one could write a wrapper to be used with + `ProcessPoolExecutor <https://docs.python.org/3/library/concurrent.futures.html>`_, + `Dask.distributed <https://distributed.dask.org/>`_, or similar libraries: + +:: + + >>> from tblib import pickling_support + >>> def wrapper(func, *args, **kwargs): + ... try: + ... return func(*args, **kwargs) + ... except Exception as e: + ... pickling_support.install(e) + ... raise + What if we have a local stack, does it show correctly ? ------------------------------------------------------- diff --git a/docs/conf.py b/docs/conf.py index 438c528..f6a4e14 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,7 +20,7 @@ project = 'tblib' year = '2013-2019' author = 'Ionel Cristian Mărieș' copyright = '{0}, {1}'.format(year, author) -version = release = '1.5.0' +version = release = '1.6.0' pygments_style = 'trac' templates_path = ['.'] @@ -25,7 +25,7 @@ def read(*names, **kwargs): setup( name='tblib', - version='1.5.0', + version='1.6.0', license='BSD-2-Clause', description='Traceback serialization library.', long_description='%s\n%s' % ( diff --git a/src/tblib.egg-info/PKG-INFO b/src/tblib.egg-info/PKG-INFO index 672d955..7f8efc3 100644 --- a/src/tblib.egg-info/PKG-INFO +++ b/src/tblib.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.2 Name: tblib -Version: 1.5.0 +Version: 1.6.0 Summary: Traceback serialization library. Home-page: https://github.com/ionelmc/python-tblib Author: Ionel Cristian Mărieș @@ -15,7 +15,7 @@ Description: ======== - Traceback serialization library. + Serialization library for Exceptions and Tracebacks. * Free software: BSD license @@ -27,6 +27,8 @@ Description: ======== * Create traceback objects from strings (the ``from_string`` method). *No pickling is used*. * Serialize tracebacks to/from plain dicts (the ``from_dict`` and ``to_dict`` methods). *No pickling is used*. * Raise the tracebacks created from the aforementioned sources. + * Pickle an Exception together with its traceback and exception chain + (``raise ... from ...``) *(Python 3 only)* **Again, note that using the pickle support is completely optional. You are solely responsible for security problems should you decide to use the pickle support.** @@ -91,8 +93,8 @@ Description: ======== >>> len(s3) > 1 True - Unpickling - ~~~~~~~~~~ + Unpickling tracebacks + ~~~~~~~~~~~~~~~~~~~~~ :: @@ -154,6 +156,92 @@ Description: ======== raise Exception('fail') Exception: fail + Pickling Exceptions together with their traceback and chain (Python 3 only) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :: + + >>> try: # doctest: +SKIP + ... try: + ... 1 / 0 + ... except Exception as e: + ... raise Exception("foo") from e + ... except Exception as e: + ... s = pickle.dumps(e) + >>> raise pickle.loads(s) # doctest: +SKIP + Traceback (most recent call last): + File "<doctest README.rst[16]>", line 3, in <module> + 1 / 0 + ZeroDivisionError: division by zero + + The above exception was the direct cause of the following exception: + + Traceback (most recent call last): + File "<doctest README.rst[17]>", line 1, in <module> + raise pickle.loads(s) + File "<doctest README.rst[16]>", line 5, in <module> + raise Exception("foo") from e + Exception: foo + + BaseException subclasses defined after calling ``pickling_support.install()`` will + **not** retain their traceback and exception chain pickling. + To cover custom Exceptions, there are three options: + + 1. Use ``@pickling_support.install`` as a decorator for each custom Exception + + .. code-block:: python + + >>> from tblib import pickling_support + >>> # Declare all imports of your package's dependencies + >>> import numpy # doctest: +SKIP + + >>> pickling_support.install() # install for all modules imported so far + + >>> @pickling_support.install + ... class CustomError(Exception): + ... pass + + Eventual subclasses of ``CustomError`` will need to be decorated again. + + 2. Invoke ``pickling_support.install()`` after all modules have been imported and all + Exception subclasses have been declared + + .. code-block:: python + + >>> # Declare all imports of your package's dependencies + >>> import numpy # doctest: +SKIP + >>> from tblib import pickling_support + + >>> # Declare your own custom Exceptions + >>> class CustomError(Exception): + ... pass + + >>> # Finally, install tblib + >>> pickling_support.install() + + 3. Selectively install tblib for Exception instances just before they are pickled + + .. code-block:: python + + pickling_support.install(<Exception instance>, [Exception instance], ...) + + The above will install tblib pickling for all listed exceptions as well as any other + exceptions in their exception chains. + + For example, one could write a wrapper to be used with + `ProcessPoolExecutor <https://docs.python.org/3/library/concurrent.futures.html>`_, + `Dask.distributed <https://distributed.dask.org/>`_, or similar libraries: + + :: + + >>> from tblib import pickling_support + >>> def wrapper(func, *args, **kwargs): + ... try: + ... return func(*args, **kwargs) + ... except Exception as e: + ... pickling_support.install(e) + ... raise + What if we have a local stack, does it show correctly ? ------------------------------------------------------- @@ -581,6 +669,13 @@ Description: ======== Changelog ========= + 1.6.0 (2019-12-07) + ~~~~~~~~~~~~~~~~~~ + + * When pickling an Exception, also pickle its traceback and the Exception chain + (``raise ... from ...``). Contributed by Guido Imperiale in + `#53 <https://github.com/ionelmc/python-tblib/issues/53>`_. + 1.5.0 (2019-10-23) ~~~~~~~~~~~~~~~~~~ diff --git a/src/tblib.egg-info/SOURCES.txt b/src/tblib.egg-info/SOURCES.txt index 1bc1caa..a913451 100644 --- a/src/tblib.egg-info/SOURCES.txt +++ b/src/tblib.egg-info/SOURCES.txt @@ -44,4 +44,5 @@ src/tblib.egg-info/top_level.txt tests/badmodule.py tests/badsyntax.py tests/examples.py -tests/test_issue30.py
\ No newline at end of file +tests/test_issue30.py +tests/test_pickle_exception.py
\ No newline at end of file diff --git a/src/tblib/__init__.py b/src/tblib/__init__.py index db17dd8..163fc62 100644 --- a/src/tblib/__init__.py +++ b/src/tblib/__init__.py @@ -15,7 +15,7 @@ except ImportError: if not tb_set_next and not tproxy: raise ImportError("Cannot use tblib. Runtime not supported.") -__version__ = '1.5.0' +__version__ = '1.6.0' __all__ = 'Traceback', PY3 = sys.version_info[0] == 3 diff --git a/src/tblib/pickling_support.py b/src/tblib/pickling_support.py index 28c4d11..cf6e390 100644 --- a/src/tblib/pickling_support.py +++ b/src/tblib/pickling_support.py @@ -1,12 +1,14 @@ -try: - import copy_reg -except ImportError: - import copyreg as copy_reg +import sys from types import TracebackType from . import Frame from . import Traceback +if sys.version_info.major >= 3: + import copyreg +else: + import copy_reg as copyreg + def unpickle_traceback(tb_frame, tb_lineno, tb_next): ret = object.__new__(Traceback) @@ -20,5 +22,66 @@ def pickle_traceback(tb): return unpickle_traceback, (Frame(tb.tb_frame), tb.tb_lineno, tb.tb_next and Traceback(tb.tb_next)) -def install(): - copy_reg.pickle(TracebackType, pickle_traceback) +def unpickle_exception(func, args, cause, tb): + inst = func(*args) + inst.__cause__ = cause + inst.__traceback__ = tb + return inst + + +def pickle_exception(obj): + # All exceptions, unlike generic Python objects, define __reduce_ex__ + # __reduce_ex__(4) should be no different from __reduce_ex__(3). + # __reduce_ex__(5) could bring benefits in the unlikely case the exception + # directly contains buffers, but PickleBuffer objects will cause a crash when + # running on protocol=4, and there's no clean way to figure out the current + # protocol from here. Note that any object returned by __reduce_ex__(3) will + # still be pickled with protocol 5 if pickle.dump() is running with it. + rv = obj.__reduce_ex__(3) + if isinstance(rv, str): + raise TypeError("str __reduce__ output is not supported") + assert isinstance(rv, tuple) and len(rv) >= 2 + + return (unpickle_exception, rv[:2] + (obj.__cause__, obj.__traceback__)) + rv[2:] + + +def _get_subclasses(cls): + # Depth-first traversal of all direct and indirect subclasses of cls + to_visit = [cls] + while to_visit: + this = to_visit.pop() + yield this + to_visit += list(this.__subclasses__()) + + +def install(*exc_classes_or_instances): + copyreg.pickle(TracebackType, pickle_traceback) + + if sys.version_info.major < 3: + # Dummy decorator? + if len(exc_classes_or_instances) == 1: + exc = exc_classes_or_instances[0] + if isinstance(exc, type) and issubclass(exc, BaseException): + return exc + return + + if not exc_classes_or_instances: + for exception_cls in _get_subclasses(BaseException): + copyreg.pickle(exception_cls, pickle_exception) + return + + for exc in exc_classes_or_instances: + if isinstance(exc, BaseException): + while exc is not None: + copyreg.pickle(type(exc), pickle_exception) + exc = exc.__cause__ + elif isinstance(exc, type) and issubclass(exc, BaseException): + copyreg.pickle(exc, pickle_exception) + # Allow using @install as a decorator for Exception classes + if len(exc_classes_or_instances) == 1: + return exc + else: + raise TypeError( + "Expected subclasses or instances of BaseException, got %s" + % (type(exc)) + ) diff --git a/tests/test_pickle_exception.py b/tests/test_pickle_exception.py new file mode 100644 index 0000000..18a018c --- /dev/null +++ b/tests/test_pickle_exception.py @@ -0,0 +1,98 @@ +try: + import copyreg +except ImportError: + # Python 2 + import copy_reg as copyreg + +import pickle +import sys + +import pytest + +import tblib.pickling_support + +has_python3 = sys.version_info.major >= 3 + + +@pytest.fixture +def clear_dispatch_table(): + bak = copyreg.dispatch_table.copy() + copyreg.dispatch_table.clear() + yield + copyreg.dispatch_table.clear() + copyreg.dispatch_table.update(bak) + + +class CustomError(Exception): + pass + + +@pytest.mark.parametrize( + "protocol", [None] + list(range(1, pickle.HIGHEST_PROTOCOL + 1)) +) +@pytest.mark.parametrize("how", ["global", "instance", "class"]) +def test_install(clear_dispatch_table, how, protocol): + if how == "global": + tblib.pickling_support.install() + elif how == "class": + tblib.pickling_support.install(CustomError, ZeroDivisionError) + + try: + try: + 1 / 0 + except Exception as e: + # Python 3 only syntax + # raise CustomError("foo") from e + new_e = CustomError("foo") + if has_python3: + new_e.__cause__ = e + raise new_e + except Exception as e: + exc = e + else: + assert False + + # Populate Exception.__dict__, which is used in some cases + exc.x = 1 + if has_python3: + exc.__cause__.x = 2 + + if how == "instance": + tblib.pickling_support.install(exc) + if protocol: + exc = pickle.loads(pickle.dumps(exc, protocol=protocol)) + + assert isinstance(exc, CustomError) + assert exc.args == ("foo",) + assert exc.x == 1 + if has_python3: + assert exc.__traceback__ is not None + assert isinstance(exc.__cause__, ZeroDivisionError) + assert exc.__cause__.__traceback__ is not None + assert exc.__cause__.x == 2 + assert exc.__cause__.__cause__ is None + + +@tblib.pickling_support.install +class RegisteredError(Exception): + pass + + +def test_install_decorator(): + with pytest.raises(RegisteredError) as ewrap: + raise RegisteredError("foo") + exc = ewrap.value + exc.x = 1 + exc = pickle.loads(pickle.dumps(exc)) + + assert isinstance(exc, RegisteredError) + assert exc.args == ("foo",) + assert exc.x == 1 + if has_python3: + assert exc.__traceback__ is not None + + +@pytest.mark.skipif(sys.version_info[0] < 3, reason="No checks done in Python 2") +def test_install_typeerror(): + with pytest.raises(TypeError): + tblib.pickling_support.install("foo") |