From 674ad1f9c1acc452eeb89e478b295c94ee3796ca Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Mon, 20 Jun 2016 11:01:26 -0700 Subject: Import python-dugong_3.7+dfsg.orig.tar.bz2 [dgit import orig python-dugong_3.7+dfsg.orig.tar.bz2] --- Changes.rst | 180 ++++ LICENSE | 244 +++++ PKG-INFO | 113 +++ README.rst | 95 ++ dugong.egg-info/PKG-INFO | 113 +++ dugong.egg-info/SOURCES.txt | 78 ++ dugong.egg-info/dependency_links.txt | 1 + dugong.egg-info/top_level.txt | 1 + dugong.egg-info/zip-safe | 1 + dugong/__init__.py | 1749 ++++++++++++++++++++++++++++++++++ examples/extract_links.py | 100 ++ examples/httpcat.py | 72 ++ examples/pipeline1.py | 81 ++ rst/_templates/localtoc.html | 2 + rst/api.rst | 101 ++ rst/conf.py | 30 + rst/coroutines.rst | 276 ++++++ rst/index.rst | 23 + rst/intro.rst | 8 + rst/issues.rst | 11 + rst/tutorial.rst | 296 ++++++ rst/whatsnew.rst | 6 + setup.cfg | 12 + setup.py | 90 ++ test/ca.crt | 20 + test/ca.key | 165 ++++ test/conftest.py | 65 ++ test/pytest.ini | 2 + test/pytest_checklogs.py | 130 +++ test/server.crt | 22 + test/server.key | 165 ++++ test/test_aio.py | 66 ++ test/test_dugong.py | 1174 +++++++++++++++++++++++ test/test_examples.py | 80 ++ 34 files changed, 5572 insertions(+) create mode 100644 Changes.rst create mode 100644 LICENSE create mode 100644 PKG-INFO create mode 100644 README.rst create mode 100644 dugong.egg-info/PKG-INFO create mode 100644 dugong.egg-info/SOURCES.txt create mode 100644 dugong.egg-info/dependency_links.txt create mode 100644 dugong.egg-info/top_level.txt create mode 100644 dugong.egg-info/zip-safe create mode 100644 dugong/__init__.py create mode 100755 examples/extract_links.py create mode 100755 examples/httpcat.py create mode 100755 examples/pipeline1.py create mode 100644 rst/_templates/localtoc.html create mode 100644 rst/api.rst create mode 100644 rst/conf.py create mode 100644 rst/coroutines.rst create mode 100644 rst/index.rst create mode 100644 rst/intro.rst create mode 100644 rst/issues.rst create mode 100644 rst/tutorial.rst create mode 100644 rst/whatsnew.rst create mode 100644 setup.cfg create mode 100755 setup.py create mode 100644 test/ca.crt create mode 100644 test/ca.key create mode 100644 test/conftest.py create mode 100644 test/pytest.ini create mode 100644 test/pytest_checklogs.py create mode 100644 test/server.crt create mode 100644 test/server.key create mode 100644 test/test_aio.py create mode 100755 test/test_dugong.py create mode 100755 test/test_examples.py diff --git a/Changes.rst b/Changes.rst new file mode 100644 index 0000000..7005d90 --- /dev/null +++ b/Changes.rst @@ -0,0 +1,180 @@ +.. currentmodule:: dugong + +Relase 3.7 (2016-06-20) +======================= + +* Dugong now supports server responses that specify just ``Connection: + close`` instead of providing the response length or using chunked + encoding. + +* Dugong now honors the `~ssl.SSLContext.check_hostname` attribute of + `~ssl.SSLContext` objects. + +Release 3.6 (2016-04-23) +======================== + +* Dugong now uses semantic versioning. This means that + backwards-incompatible versions (i.e., versions that change the + existing API in some way) will be reflected in an increase of the + major version number, i.e. the next backwards-incompatible version + will have version 4.0. + +* Various minor bugfixes. + +Release 3.5 (2015-01-31) +======================== + +* The `is_temp_network_error` function now knows about more kinds of + temporary errors, in particular also about those that do not have a + corresponding Python exception (e.g. no route to host). + +Release 3.4 (2014-11-29) +======================== + +* The :file:`examples/extract_links.py` script is now Python 3.3 + compatible again. + +* Dugong now has proper proxy support for plain HTTP + connections. Thanks to Vincent Bernat for the patch. + +* Dugong now raises `DNSUnavailable` or `HostnameNotResolvable` + exceptions instead of the more generic `socket.gaierror` and + `socket.herror` exceptions where possible. + +Release 3.3 (2014-08-06) +======================== + +* It was possible for some methods to raise `BrokenPipeError`, or + `ConnectionResetError` instead of `ConnectionClosed` (especially + under FreeBSD). This has now been fixed. + +* It was possible for methods that need to read data from the server + to raise `AssertionError` in some circumstances. This has been + fixed. + +Release 3.2 (2014-07-27) +======================== + +* A `HTTPConnection` instance can now be used as a context manager. + +* If a connection is closed unexpectedly while request body data is + being written to the server (i.e., during a call to + `HTTPConnection.write` or `HTTPConnection.co_write`), dugong now + pretends that the body has been sent to the server completely (while + still raising `ConnectionClosed`). + + This makes it possible to catch the exception and nevertheless call + `~HTTPConnection.read_response` (or + `~HTTPConnection.co_read_response`) to try to read an error response + that the server may have sent during the upload (if no response has + been received, `ConnectionClosed` will be raised again). + +* `is_temp_network_error` now actively tries to distinguish between + permanent and temporary name resolution problems by attempting to + resolve a number of test hostnames. + +* `HTTPConnection` has a new `~HTTPConnection.timeout` + attribute. Regular `HTTPConnection` methods (i.e., no coroutines) + will now raise `ConnectionTimedOut` if no data could be send or + received for *timeout* seconds. + +Release 3.1 (2014-06-28) +======================== + +* Fixed a problem with some testcases failing with a BrokenPipeError. + +* Fixed a bug that, in some cases, resulted in additional ``\0`` bytes + being appended at the end of the response body, or in an incorrect + `InvalidResponse` exception being raised. + +* When trying to continue reading or writing body data after calling + `HTTPConnection.disconnect`, dugong now raises `ConnectionClosed` + instead of `AttributeError`. + +Release 3.0, (2014-04-20) +========================= + +* Major version bump because of backwards incompatible changes. + +* Added `HTTPConnection.read_raw` method. + +* The `PollNeeded` class now uses the `!select.POLLIN` and + `!select.POLLOUT` constants instead of `!select.EPOLLIN` and + `!select.EPOLLOUT` to signal what kind of I/O needs to be + performed. This makes dugong compatible with systems lacking epoll + (e.g. FreeBSD). + +* The unit tests now check if the host is reachable before trying to + run the example scripts. This avoids bogus test errors if + there is no internet connection or if the remote host is down. + (issue #7). + + +Release 2.2 (2014-03-14) +======================== + +* Unittests requiring the `asyncio` module are now skipped if this + module is not available. + + +Release 2.1 (2014-03-11) +======================== + +* Fixed a problem where data was not sent to the server if the syscall + was interrupted by a signal. + +* It is no longer necessary to read from response body at least once + even if has zero length. + +* `PollNeeded.poll` now uses `select.poll` instead of + `select.select`. This avoids a "filedescriptor out of range" + exception that may be raised by `select.select` when the + filedescriptor exceeds some system-specific value. + + +Release 2.0 (2014-02-23) +======================== + +* Renamed module from *httpio* to *dugong*. + +* The coroutine based API was completely overhauled. + +* Introduced `BodyFollowing` class for use with *body* parameter of + `~HTTPConnection.send_request` method. + +* `~HTTPConnection.send_request` now returns a `HTTPResponse` instance + instead of a tuple. + +* The :meth:`!HTTPConnection.get_current_response` method has been removed. + +* The :meth:`!HTTPConnection.fileno` method has been removed. + +* Added `CaseInsensitiveDict` class. + +* `~HTTPConnection.send_request` now converts the *header* parameter + to a `CaseInsensitiveDict`. + +* `~HTTPConnection.send_request` now automatically generates a + ``Content-MD5`` header when the body is passed in as a bytes-like + object. + +* `HTTPConnection.read` now accepts `None` for the *len_* parameter. + +* `HTTPConnection` instances now support a bare-bones `io.IOBase` + interface so that they can be combined with `io.TextIOWrapper` to + read text response bodies. + +* The :meth:`!HTTPConnection.close` method was renamed to + `HTTPConnection.disconnect` to prevent confusion related to the + ``closed`` attribute (which may be `True` if the connection is + established, but there is no active response body). + +* Repeatedly trying to read more response data after the response body + has been read completely no longer results in `StateError` being + raised, but simply returns ``b''``. + + +Release 1.0 (2013-07-13) +======================== + +* Initial release. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..49c2cac --- /dev/null +++ b/LICENSE @@ -0,0 +1,244 @@ +====================================== + License for the Dugong Python module +====================================== + +This module is copyright (C) Nikolaus Rath . + +It may be distributed under the terms of the Python Software +Foundation License Version 2, reproduced below. + +The included CaseInsensitiveDict implementation is copyright 2013 +Kenneth Reitz and licensed under the Apache License, Version 2.0 +(http://www.apache.org/licenses/LICENSE-2.0, reproduced below.) + + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +============================================ + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved" are retained +in Python alone or in any derivative version prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. + + + +Apache License, Version 2.0 +=========================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + diff --git a/PKG-INFO b/PKG-INFO new file mode 100644 index 0000000..dedc06a --- /dev/null +++ b/PKG-INFO @@ -0,0 +1,113 @@ +Metadata-Version: 1.1 +Name: dugong +Version: 3.7 +Summary: A HTTP 1.1 client module supporting asynchronous IO, pipelining and `Expect: 100-continue`. Designed for RESTful protocols. +Home-page: https://bitbucket.org/nikratio/python-dugong +Author: Nikolaus Rath +Author-email: Nikolaus@rath.org +License: PSF +Description: ========================== + The Python Dugong Module + ========================== + + .. default-role:: code + + .. start-intro + + The Python Dugong module provides an API for communicating with HTTP + 1.1 servers. It is an alternative to the standard library's + `http.client` (formerly *httplib*) module. In contrast to + `http.client`, Dugong: + + - allows you to send multiple requests right after each other without + having to read the responses first. + + - supports waiting for 100-continue before sending the request body. + + - raises an exception instead of silently delivering partial data if the + connection is closed before all data has been received. + + - raises one specific exception (`ConnectionClosed`) if the connection + has been closed (while `http.client` connection may raise any of + `BrokenPipeError`, `~http.client.BadStatusLine`, + `ConnectionAbortedError`, `ConnectionResetError`, + `~http.client.IncompleteRead` or simply return ``''`` on read) + + - supports non-blocking, asynchronous operation and is compatible with + the asyncio_ module. + + - can in most cases distinguish between an unavailable DNS server and + an unresolvable hostname. + + - is not compatible with old HTTP 0.9 or 1.0 servers. + + All request and response headers are represented as `str`, but must be + encodable in latin1. Request and response body must be `bytes-like + objects`_ or binary streams. + + Dugong requires Python 3.3 or newer. + + .. _`bytes-like objects`: http://docs.python.org/3/glossary.html#term-bytes-like-object + .. _asyncio: http://docs.python.org/3.4/library/asyncio.html + + + Installation + ============ + + As usual: download the tarball from PyPi_, extract it, and run :: + + # python3 setup.py install [--user] + + To run the self-tests, install `py.test`_ with the `pytest-catchlog`_ + plugin and run :: + + # python3 -m pytest test/ + + .. _PyPi: https://pypi.python.org/pypi/dugong/#downloads + .. _py.test: http://www.pytest.org/ + .. _pytest-catchlog: https://github.com/eisensheng/pytest-catchlog + + + Getting Help + ============ + + The documentation can be `read online`__ and is also included in the + *doc/html* directory of the dugong tarball. + + Please report any bugs on the `BitBucket issue tracker`_. For discussion and + questions, please subscribe to the `dugong mailing list`_. + + .. __: http://pythonhosted.org/dugong/ + .. _dugong mailing list: https://groups.google.com/d/forum/python-dugong + .. _`BitBucket issue tracker`: https://bitbucket.org/nikratio/python-dugong/issues + + + Development Status + ================== + + The Dugong API is not yet stable and may change from one release to + the other. Starting with version 3.5, Dugong uses semantic + versioning. This means changes in the API will be reflected in an + increase of the major version number, i.e. the next + backwards-incompatible version will be 4.0. Projects designed for + e.g. version 3.5 of Dugong are thus recommended to declare a + dependency on ``dugong >= 3.5, < 4.0``. + + + Contributing + ============ + + The LLFUSE source code is available both on GitHub_ and BitBucket_. + + .. _BitBucket: https://bitbucket.org/nikratio/python-dugong/ + .. _GitHub: https://github.com/python-dugong/main + +Keywords: http +Platform: UNKNOWN +Classifier: Programming Language :: Python :: 3 +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Python Software Foundation License +Classifier: Topic :: Internet :: WWW/HTTP +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Provides: dugong diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..8769189 --- /dev/null +++ b/README.rst @@ -0,0 +1,95 @@ +========================== + The Python Dugong Module +========================== + +.. default-role:: code + +.. start-intro + +The Python Dugong module provides an API for communicating with HTTP +1.1 servers. It is an alternative to the standard library's +`http.client` (formerly *httplib*) module. In contrast to +`http.client`, Dugong: + +- allows you to send multiple requests right after each other without + having to read the responses first. + +- supports waiting for 100-continue before sending the request body. + +- raises an exception instead of silently delivering partial data if the + connection is closed before all data has been received. + +- raises one specific exception (`ConnectionClosed`) if the connection + has been closed (while `http.client` connection may raise any of + `BrokenPipeError`, `~http.client.BadStatusLine`, + `ConnectionAbortedError`, `ConnectionResetError`, + `~http.client.IncompleteRead` or simply return ``''`` on read) + +- supports non-blocking, asynchronous operation and is compatible with + the asyncio_ module. + +- can in most cases distinguish between an unavailable DNS server and + an unresolvable hostname. + +- is not compatible with old HTTP 0.9 or 1.0 servers. + +All request and response headers are represented as `str`, but must be +encodable in latin1. Request and response body must be `bytes-like +objects`_ or binary streams. + +Dugong requires Python 3.3 or newer. + +.. _`bytes-like objects`: http://docs.python.org/3/glossary.html#term-bytes-like-object +.. _asyncio: http://docs.python.org/3.4/library/asyncio.html + + +Installation +============ + +As usual: download the tarball from PyPi_, extract it, and run :: + + # python3 setup.py install [--user] + +To run the self-tests, install `py.test`_ with the `pytest-catchlog`_ +plugin and run :: + + # python3 -m pytest test/ + +.. _PyPi: https://pypi.python.org/pypi/dugong/#downloads +.. _py.test: http://www.pytest.org/ +.. _pytest-catchlog: https://github.com/eisensheng/pytest-catchlog + + +Getting Help +============ + +The documentation can be `read online`__ and is also included in the +*doc/html* directory of the dugong tarball. + +Please report any bugs on the `BitBucket issue tracker`_. For discussion and +questions, please subscribe to the `dugong mailing list`_. + +.. __: http://pythonhosted.org/dugong/ +.. _dugong mailing list: https://groups.google.com/d/forum/python-dugong +.. _`BitBucket issue tracker`: https://bitbucket.org/nikratio/python-dugong/issues + + +Development Status +================== + +The Dugong API is not yet stable and may change from one release to +the other. Starting with version 3.5, Dugong uses semantic +versioning. This means changes in the API will be reflected in an +increase of the major version number, i.e. the next +backwards-incompatible version will be 4.0. Projects designed for +e.g. version 3.5 of Dugong are thus recommended to declare a +dependency on ``dugong >= 3.5, < 4.0``. + + +Contributing +============ + +The LLFUSE source code is available both on GitHub_ and BitBucket_. + +.. _BitBucket: https://bitbucket.org/nikratio/python-dugong/ +.. _GitHub: https://github.com/python-dugong/main diff --git a/dugong.egg-info/PKG-INFO b/dugong.egg-info/PKG-INFO new file mode 100644 index 0000000..dedc06a --- /dev/null +++ b/dugong.egg-info/PKG-INFO @@ -0,0 +1,113 @@ +Metadata-Version: 1.1 +Name: dugong +Version: 3.7 +Summary: A HTTP 1.1 client module supporting asynchronous IO, pipelining and `Expect: 100-continue`. Designed for RESTful protocols. +Home-page: https://bitbucket.org/nikratio/python-dugong +Author: Nikolaus Rath +Author-email: Nikolaus@rath.org +License: PSF +Description: ========================== + The Python Dugong Module + ========================== + + .. default-role:: code + + .. start-intro + + The Python Dugong module provides an API for communicating with HTTP + 1.1 servers. It is an alternative to the standard library's + `http.client` (formerly *httplib*) module. In contrast to + `http.client`, Dugong: + + - allows you to send multiple requests right after each other without + having to read the responses first. + + - supports waiting for 100-continue before sending the request body. + + - raises an exception instead of silently delivering partial data if the + connection is closed before all data has been received. + + - raises one specific exception (`ConnectionClosed`) if the connection + has been closed (while `http.client` connection may raise any of + `BrokenPipeError`, `~http.client.BadStatusLine`, + `ConnectionAbortedError`, `ConnectionResetError`, + `~http.client.IncompleteRead` or simply return ``''`` on read) + + - supports non-blocking, asynchronous operation and is compatible with + the asyncio_ module. + + - can in most cases distinguish between an unavailable DNS server and + an unresolvable hostname. + + - is not compatible with old HTTP 0.9 or 1.0 servers. + + All request and response headers are represented as `str`, but must be + encodable in latin1. Request and response body must be `bytes-like + objects`_ or binary streams. + + Dugong requires Python 3.3 or newer. + + .. _`bytes-like objects`: http://docs.python.org/3/glossary.html#term-bytes-like-object + .. _asyncio: http://docs.python.org/3.4/library/asyncio.html + + + Installation + ============ + + As usual: download the tarball from PyPi_, extract it, and run :: + + # python3 setup.py install [--user] + + To run the self-tests, install `py.test`_ with the `pytest-catchlog`_ + plugin and run :: + + # python3 -m pytest test/ + + .. _PyPi: https://pypi.python.org/pypi/dugong/#downloads + .. _py.test: http://www.pytest.org/ + .. _pytest-catchlog: https://github.com/eisensheng/pytest-catchlog + + + Getting Help + ============ + + The documentation can be `read online`__ and is also included in the + *doc/html* directory of the dugong tarball. + + Please report any bugs on the `BitBucket issue tracker`_. For discussion and + questions, please subscribe to the `dugong mailing list`_. + + .. __: http://pythonhosted.org/dugong/ + .. _dugong mailing list: https://groups.google.com/d/forum/python-dugong + .. _`BitBucket issue tracker`: https://bitbucket.org/nikratio/python-dugong/issues + + + Development Status + ================== + + The Dugong API is not yet stable and may change from one release to + the other. Starting with version 3.5, Dugong uses semantic + versioning. This means changes in the API will be reflected in an + increase of the major version number, i.e. the next + backwards-incompatible version will be 4.0. Projects designed for + e.g. version 3.5 of Dugong are thus recommended to declare a + dependency on ``dugong >= 3.5, < 4.0``. + + + Contributing + ============ + + The LLFUSE source code is available both on GitHub_ and BitBucket_. + + .. _BitBucket: https://bitbucket.org/nikratio/python-dugong/ + .. _GitHub: https://github.com/python-dugong/main + +Keywords: http +Platform: UNKNOWN +Classifier: Programming Language :: Python :: 3 +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: Python Software Foundation License +Classifier: Topic :: Internet :: WWW/HTTP +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Provides: dugong diff --git a/dugong.egg-info/SOURCES.txt b/dugong.egg-info/SOURCES.txt new file mode 100644 index 0000000..b85a6a4 --- /dev/null +++ b/dugong.egg-info/SOURCES.txt @@ -0,0 +1,78 @@ +Changes.rst +LICENSE +README.rst +setup.cfg +setup.py +./dugong/__init__.py +doc/html/.buildinfo +doc/html/api.html +doc/html/coroutine.html +doc/html/coroutines.html +doc/html/expect100.html +doc/html/genindex.html +doc/html/index.html +doc/html/intro.html +doc/html/issues.html +doc/html/objects.inv +doc/html/search.html +doc/html/searchindex.js +doc/html/tutorial.html +doc/html/whatsnew.html +doc/html/_modules/dugong.html +doc/html/_modules/index.html +doc/html/_sources/api.txt +doc/html/_sources/coroutine.txt +doc/html/_sources/coroutines.txt +doc/html/_sources/expect100.txt +doc/html/_sources/index.txt +doc/html/_sources/intro.txt +doc/html/_sources/issues.txt +doc/html/_sources/tutorial.txt +doc/html/_sources/whatsnew.txt +doc/html/_static/ajax-loader.gif +doc/html/_static/basic.css +doc/html/_static/comment-bright.png +doc/html/_static/comment-close.png +doc/html/_static/comment.png +doc/html/_static/default.css +doc/html/_static/doctools.js +doc/html/_static/down-pressed.png +doc/html/_static/down.png +doc/html/_static/file.png +doc/html/_static/jquery.js +doc/html/_static/minus.png +doc/html/_static/plus.png +doc/html/_static/pygments.css +doc/html/_static/searchtools.js +doc/html/_static/sidebar.js +doc/html/_static/underscore.js +doc/html/_static/up-pressed.png +doc/html/_static/up.png +doc/html/_static/websupport.js +dugong.egg-info/PKG-INFO +dugong.egg-info/SOURCES.txt +dugong.egg-info/dependency_links.txt +dugong.egg-info/top_level.txt +dugong.egg-info/zip-safe +examples/extract_links.py +examples/httpcat.py +examples/pipeline1.py +rst/api.rst +rst/conf.py +rst/coroutines.rst +rst/index.rst +rst/intro.rst +rst/issues.rst +rst/tutorial.rst +rst/whatsnew.rst +rst/_templates/localtoc.html +test/ca.crt +test/ca.key +test/conftest.py +test/pytest.ini +test/pytest_checklogs.py +test/server.crt +test/server.key +test/test_aio.py +test/test_dugong.py +test/test_examples.py \ No newline at end of file diff --git a/dugong.egg-info/dependency_links.txt b/dugong.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/dugong.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/dugong.egg-info/top_level.txt b/dugong.egg-info/top_level.txt new file mode 100644 index 0000000..21bc003 --- /dev/null +++ b/dugong.egg-info/top_level.txt @@ -0,0 +1 @@ +dugong diff --git a/dugong.egg-info/zip-safe b/dugong.egg-info/zip-safe new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/dugong.egg-info/zip-safe @@ -0,0 +1 @@ + diff --git a/dugong/__init__.py b/dugong/__init__.py new file mode 100644 index 0000000..92e0a52 --- /dev/null +++ b/dugong/__init__.py @@ -0,0 +1,1749 @@ +''' +dugong.py - Python HTTP Client Module + +Copyright © 2014 Nikolaus Rath + +This module may be distributed under the terms of the Python Software Foundation +License Version 2. + +The CaseInsensitiveDict implementation is copyright 2013 Kenneth Reitz and +licensed under the Apache License, Version 2.0 +(http://www.apache.org/licenses/LICENSE-2.0) +''' + +import socket +import logging +import errno +import ssl +import hashlib +from inspect import getdoc +import textwrap +from base64 import b64encode +from collections import deque +from collections.abc import MutableMapping, Mapping +import email +import email.policy +from http.client import (HTTPS_PORT, HTTP_PORT, NO_CONTENT, NOT_MODIFIED) +from select import POLLIN, POLLOUT +import select +import sys + +try: + import asyncio +except ImportError: + asyncio = None + +# Enums are only available in 3.4 and newer +try: + from enum import Enum +except ImportError: + Enum = object + +__version__ = '3.7' + +log = logging.getLogger(__name__) + +#: Internal buffer size +BUFFER_SIZE = 64*1024 + +#: Maximal length of HTTP status line. If the server sends a line longer than +#: this value, `InvalidResponse` will be raised. +MAX_LINE_SIZE = BUFFER_SIZE-1 + +#: Maximal length of a response header (i.e., for all header +#: lines together). If the server sends a header segment longer than +#: this value, `InvalidResponse` will be raised. +MAX_HEADER_SIZE = BUFFER_SIZE-1 + +class Symbol: + ''' + A symbol instance represents a specific state. Its value is + not relevant, as it should only ever be assigned to or compared + with other variables. + ''' + __slots__ = [ 'name' ] + + def __init__(self, name): + self.name = name + + def __str__(self): + return self.name + + def __repr__(self): + return '<%s>' % (self.name,) + +class Encodings(Enum): + CHUNKED = 1 + IDENTITY = 2 + +#: Sentinel for `HTTPConnection._out_remaining` to indicate that +#: we're waiting for a 100-continue response from the server +WAITING_FOR_100c = Symbol('WAITING_FOR_100c') + +#: Sentinel for `HTTPConnection._in_remaining` to indicate that +#: the response body cannot be read and that an exception +#: (stored in the `HTTPConnection._encoding` attribute) should +#: be raised. +RESPONSE_BODY_ERROR = Symbol('RESPONSE_BODY_ERROR') + +#: Sentinel for `HTTPConnection._in_remaining` to indicate that +#: we should read until EOF (i.e., no keep-alive) +READ_UNTIL_EOF = Symbol('READ_UNTIL_EOF') + +#: Sequence of ``(hostname, port)`` tuples that are used by +#: dugong to distinguish between permanent and temporary name +#: resolution problems. +DNS_TEST_HOSTNAMES=(('www.google.com', 80), + ('www.iana.org', 80), + ('C.root-servers.org', 53)) + +class PollNeeded(tuple): + ''' + This class encapsulates the requirements for a IO operation to continue. + `PollNeeded` instances are typically yielded by coroutines. + ''' + + __slots__ = () + + def __new__(self, fd, mask): + return tuple.__new__(self, (fd, mask)) + + @property + def fd(self): + '''File descriptor that the IO operation depends on''' + + return self[0] + + @property + def mask(self): + '''Event mask specifiying the type of required IO + + This attribute defines what type of IO the provider of the `PollNeeded` + instance needs to perform on *fd*. It is expected that, when *fd* is + ready for IO of the specified type, operation will continue without + blocking. + + The type of IO is specified as a :ref:`poll ` + compatible event mask, i.e. a bitwise combination of `!select.POLLIN` + and `!select.POLLOUT`. + ''' + + return self[1] + + def poll(self, timeout=None): + '''Wait until fd is ready for requested IO + + This is a convenince function that uses `~select.poll` to wait until + `.fd` is ready for requested type of IO. + + If *timeout* is specified, return `False` if the timeout is exceeded + without the file descriptor becoming ready. + ''' + + poll = select.poll() + poll.register(self.fd, self.mask) + + log.debug('calling poll') + if timeout: + return bool(poll.poll(timeout*1000)) # convert to ms + else: + return bool(poll.poll()) + + + +class HTTPResponse: + ''' + This class encapsulates information about HTTP response. Instances of this + class are returned by the `HTTPConnection.read_response` method and have + access to response status, reason, and headers. Response body data + has to be read directly from the `HTTPConnection` instance. + ''' + + def __init__(self, method, path, status, reason, headers, + length=None): + + #: HTTP Method of the request this was response is associated with + self.method = method + + #: Path of the request this was response is associated with + self.path = path + + #: HTTP status code returned by the server + self.status = status + + #: HTTP reason phrase returned by the server + self.reason = reason + + #: HTTP Response headers, a `email.message.Message` instance + self.headers = headers + + #: Length of the response body or `None` if not known. This attribute + #: contains the actual length of the *transmitted* response. That means + #: that for responses where RFC 2616 mandates that no request body + #: be sent (e.g. in response to HEAD requests or for 1xx response + #: codes) this value is zero. In these cases, the length of the body that + #: *would* have been send can be extracted from the ``Content-Length`` + #: response header. + self.length = length + + +class BodyFollowing: + ''' + Sentinel class for the *body* parameter of the + `~HTTPConnection.send_request` method. Passing an instance of this class + declares that body data is going to be provided in separate method calls. + + If no length is specified in the constructor, the body data will be send + using chunked encoding. + ''' + + __slots__ = 'length' + + def __init__(self, length=None): + #: the length of the body data that is going to be send, or `None` + #: to use chunked encoding. + self.length = length + + +class _ChunkTooLong(Exception): + ''' + Raised by `_co_readstr_until` if the requested end pattern + cannot be found within the specified byte limit. + ''' + + pass + + +class _GeneralError(Exception): + msg = 'General HTTP Error' + + def __init__(self, msg=None): + if msg: + self.msg = msg + + def __str__(self): + return self.msg + +class HostnameNotResolvable(Exception): + '''Raised if a host name does not resolve to an ip address. + + Dugong raises this exception if a resolution attempt results in a + `socket.gaierror` or `socket.herror` exception with errno + :const:`!socket.EAI_AGAIN` or :const:`!socket.EAI_NONAME`, but at least one of + the hostnames in `DNS_TEST_HOSTNAMES` can be resolved. + ''' + + def __init__(self, hostname): + self.name = hostname + + def __str__(self): + return 'Host %s does not have any ip addresses' % self.name + + +class DNSUnavailable(Exception): + '''Raised if the DNS server cannot be reached. + + Dugong raises this exception if a resolution attempt results in a + `socket.gaierror` or `socket.herror` exception with errno + :const:`!socket.EAI_AGAIN` or :const:`!socket.EAI_NONAME`, and none of the + hostnames in `DNS_TEST_HOSTNAMES` can be resolved either. + ''' + + def __init__(self, hostname): + self.name = hostname + + def __str__(self): + return 'Unable to resolve %s, DNS server unavailable.' % self.name + +class StateError(_GeneralError): + ''' + Raised when attempting an operation that doesn't make + sense in the current connection state. + ''' + + msg = 'Operation invalid in current connection state' + + +class ExcessBodyData(_GeneralError): + ''' + Raised when trying to send more data to the server than + announced. + ''' + + msg = 'Cannot send larger request body than announced' + + +class InvalidResponse(_GeneralError): + ''' + Raised if the server produced an invalid response (i.e, something + that is not proper HTTP 1.0 or 1.1). + ''' + + msg = 'Server sent invalid response' + + +class UnsupportedResponse(_GeneralError): + ''' + This exception is raised if the server produced a response that is not + supported. This should not happen for servers that are HTTP 1.1 compatible. + + If an `UnsupportedResponse` exception has been raised, this typically means + that synchronization with the server will be lost (i.e., dugong cannot + determine where the current response ends and the next response starts), so + the connection needs to be reset by calling the + :meth:`~HTTPConnection.disconnect` method. + ''' + + msg = 'Server sent unsupported response' + + +class ConnectionClosed(_GeneralError): + ''' + Raised if the connection was unexpectedly closed. + + This exception is raised also if the server declared that it will close the + connection (by sending a ``Connection: close`` header). Such responses can + still be read completely, but the next attempt to send a request or read a + response will raise the exception. To re-use the connection after the server + has closed the connection, call `HTTPConnection.reset` before further + requests are send. + + This behavior is intentional, because the caller may have already issued + other requests (i.e., used pipelining). By raising an exception, the caller + is notified that any pending requests have been lost and need to be resend. + ''' + + msg = 'connection closed unexpectedly' + + +class ConnectionTimedOut(_GeneralError): + ''' + Raised if a regular `HTTPConnection` method (i.e., no coroutine) was + unable to send or receive data for the timeout specified in the + `HTTPConnection.timeout` attribute. + ''' + + msg = 'send/recv timeout exceeded' + +class _Buffer: + ''' + This class represents a buffer with a fixed size, but varying + fill level. + ''' + + __slots__ = ('d', 'b', 'e') + + def __init__(self, size): + + #: Holds the actual data + self.d = bytearray(size) + + #: Position of the first buffered byte that has not yet + #: been consumed ("*b*eginning") + self.b = 0 + + #: Fill-level of the buffer ("*e*nd") + self.e = 0 + + def __len__(self): + '''Return amount of data ready for consumption''' + return self.e - self.b + + def clear(self): + '''Forget all buffered data''' + + self.b = 0 + self.e = 0 + + def compact(self): + '''Ensure that buffer can be filled up to its maximum size + + If part of the buffer data has been consumed, the unconsumed part is + copied to the beginning of the buffer to maximize the available space. + ''' + + if self.b == 0: + return + + log.debug('compacting buffer') + buf = memoryview(self.d)[self.b:self.e] + len_ = len(buf) + self.d = bytearray(len(self.d)) + self.d[:len_] = buf + self.b = 0 + self.e = len_ + + def exhaust(self): + '''Return (and consume) all available data''' + + if self.b == 0: + log.debug('exhausting buffer (truncating)') + # Return existing buffer after truncating it + buf = self.d + self.d = bytearray(len(self.d)) + buf[self.e:] = b'' + else: + log.debug('exhausting buffer (copying)') + buf = self.d[self.b:self.e] + + self.b = 0 + self.e = 0 + + return buf + + +class HTTPConnection: + ''' + This class encapsulates a HTTP connection. + + Methods whose name begin with ``co_`` return coroutines. Instead of + blocking, a coroutines will yield a `PollNeeded` instance that encapsulates + information about the IO operation that would block. The coroutine should be + resumed once the operation can be performed without blocking. + + `HTTPConnection` instances can be used as context managers. The + `.disconnect` method will be called on exit from the managed block. + ''' + + def __init__(self, hostname, port=None, ssl_context=None, proxy=None): + + if port is None: + if ssl_context is None: + self.port = HTTP_PORT + else: + self.port = HTTPS_PORT + else: + self.port = port + + self.ssl_context = ssl_context + self.hostname = hostname + + #: Socket object connecting to the server + self._sock = None + + #: Read-buffer + self._rbuf = _Buffer(BUFFER_SIZE) + + #: a tuple ``(hostname, port)`` of the proxy server to use or `None`. + #: Note that currently only CONNECT-style proxying is supported. + self.proxy = proxy + + #: a deque of ``(method, path, body_len)`` tuples corresponding to + #: requests whose response has not yet been read completely. Requests + #: with Expect: 100-continue will be added twice to this queue, once + #: after the request header has been sent, and once after the request + #: body data has been sent. *body_len* is `None`, or the size of the + #: **request** body that still has to be sent when using 100-continue. + self._pending_requests = deque() + + #: This attribute is `None` when a request has been sent completely. If + #: request headers have been sent, but request body data is still + #: pending, it is set to a ``(method, path, body_len)`` tuple. *body_len* + #: is the number of bytes that that still need to send, or + #: `WAITING_FOR_100c` if we are waiting for a 100 response from the server. + self._out_remaining = None + + #: Number of remaining bytes of the current response body (or current + #: chunk), `None` if there is no active response or `READ_UNTIL_EOF` if + #: we have to read until the connection is closed (i.e., we don't know + #: the content-length and keep-alive is not active). + self._in_remaining = None + + #: Transfer encoding of the active response (if any). + self._encoding = None + + #: If a regular `HTTPConnection` method is unable to send or receive + #: data for more than this period (in seconds), it will raise + #: `ConnectionTimedOut`. Coroutines are not affected by this + #: attribute. + self.timeout = None + + # Implement bare-bones `io.BaseIO` interface, so that instances + # can be wrapped in `io.TextIOWrapper` if desired. + def writable(self): + return True + def readable(self): + return True + def seekable(self): + return False + + # One could argue that the stream should be considered closed if + # there is no active response. However, this breaks TextIOWrapper + # (which fails if the stream becomes closed even after b'' has + # been read), so we just declare to be always open. + closed = False + + def connect(self): + """Connect to the remote server + + This method generally does not need to be called manually. + """ + + log.debug('start') + + if self.proxy: + log.debug('connecting to %s', self.proxy) + self._sock = create_socket(self.proxy) + if self.ssl_context: + eval_coroutine(self._co_tunnel(), self.timeout) + else: + log.debug('connecting to %s', (self.hostname, self.port)) + self._sock = create_socket((self.hostname, self.port)) + + if self.ssl_context: + log.debug('establishing ssl layer') + if (sys.version_info >= (3, 5) or + (sys.version_info >= (3, 4) and ssl.HAS_SNI)): + # Automatic hostname verification was added in 3.4, but only + # if SNI is available. In 3.5 the hostname can be checked even + # if there is no SNI support. + server_hostname = self.hostname + else: + server_hostname = None + self._sock = self.ssl_context.wrap_socket(self._sock, server_hostname=server_hostname) + + if server_hostname is None: + # Manually check hostname for Python < 3.4, or if we have + # 3.4 without SNI. + try: + ssl.match_hostname(self._sock.getpeercert(), self.hostname) + except: + self.close() + raise + + self._sock.setblocking(False) + self._rbuf.clear() + self._out_remaining = None + self._in_remaining = None + self._pending_requests = deque() + + log.debug('done') + + def _co_tunnel(self): + '''Set up CONNECT tunnel to destination server''' + + log.debug('start connecting to %s:%d', self.hostname, self.port) + + yield from self._co_send(("CONNECT %s:%d HTTP/1.0\r\n\r\n" + % (self.hostname, self.port)).encode('latin1')) + + (status, reason) = yield from self._co_read_status() + log.debug('got %03d %s', status, reason) + yield from self._co_read_header() + + if status != 200: + self.disconnect() + raise ConnectionError("Tunnel connection failed: %d %s" % (status, reason)) + + def get_ssl_peercert(self, binary_form=False): + '''Get peer SSL certificate + + If plain HTTP is used, return `None`. Otherwise, the call is delegated + to the underlying SSL sockets `~ssl.SSLSocket.getpeercert` method. + ''' + + if not self.ssl_context: + return None + else: + if not self._sock: + self.connect() + return self._sock.getpeercert() + + def get_ssl_cipher(self): + '''Get active SSL cipher + + If plain HTTP is used, return `None`. Otherwise, the call is delegated + to the underlying SSL sockets `~ssl.SSLSocket.cipher` method. + ''' + + if not self.ssl_context: + return None + else: + if not self._sock: + self.connect() + return self._sock.cipher() + + def send_request(self, method, path, headers=None, body=None, expect100=False): + '''placeholder, will be replaced dynamically''' + eval_coroutine(self.co_send_request(method, path, headers=headers, + body=body, expect100=expect100), + self.timeout) + + def co_send_request(self, method, path, headers=None, body=None, expect100=False): + '''Send a new HTTP request to the server + + The message body may be passed in the *body* argument or be sent + separately. In the former case, *body* must be a :term:`bytes-like + object`. In the latter case, *body* must be an a `BodyFollowing` + instance specifying the length of the data that will be sent. If no + length is specified, the data will be send using chunked encoding. + + *headers* should be a mapping containing the HTTP headers to be send + with the request. Multiple header lines with the same key are not + supported. It is recommended to pass a `CaseInsensitiveDict` instance, + other mappings will be converted to `CaseInsensitiveDict` automatically. + + If *body* is a provided as a :term:`bytes-like object`, a + ``Content-MD5`` header is generated automatically unless it has been + provided in *headers* already. + ''' + + log.debug('start') + + if expect100 and not isinstance(body, BodyFollowing): + raise ValueError('expect100 only allowed for separate body') + + if self._sock is None: + self.connect() + + if self._out_remaining: + raise StateError('body data has not been sent completely yet') + + if headers is None: + headers = CaseInsensitiveDict() + elif not isinstance(headers, CaseInsensitiveDict): + headers = CaseInsensitiveDict(headers) + + pending_body_size = None + if body is None: + headers['Content-Length'] = '0' + elif isinstance(body, BodyFollowing): + if body.length is None: + raise ValueError('Chunked encoding not yet supported.') + log.debug('preparing to send %d bytes of body data', body.length) + if expect100: + headers['Expect'] = '100-continue' + # Do not set _out_remaining, we must only send data once we've + # read the response. Instead, save body size in + # _pending_requests so that it can be restored by + # read_response(). + pending_body_size = body.length + self._out_remaining = (method, path, WAITING_FOR_100c) + else: + self._out_remaining = (method, path, body.length) + headers['Content-Length'] = str(body.length) + body = None + elif isinstance(body, (bytes, bytearray, memoryview)): + headers['Content-Length'] = str(len(body)) + if 'Content-MD5' not in headers: + log.debug('computing content-md5') + headers['Content-MD5'] = b64encode(hashlib.md5(body).digest()).decode('ascii') + else: + raise TypeError('*body* must be None, bytes-like or BodyFollowing') + + # Generate host header + host = self.hostname + if host.find(':') >= 0: + host = '[{}]'.format(host) + default_port = HTTPS_PORT if self.ssl_context else HTTP_PORT + if self.port == default_port: + headers['Host'] = host + else: + headers['Host'] = '{}:{}'.format(host, self.port) + + # Assemble request + headers['Accept-Encoding'] = 'identity' + if 'Connection' not in headers: + headers['Connection'] = 'keep-alive' + if self.proxy and not self.ssl_context: + gpath = "http://{}{}".format(headers['Host'], path) + else: + gpath = path + request = [ '{} {} HTTP/1.1'.format(method, gpath).encode('latin1') ] + for key, val in headers.items(): + request.append('{}: {}'.format(key, val).encode('latin1')) + request.append(b'') + + if body is not None: + request.append(body) + else: + request.append(b'') + + buf = b'\r\n'.join(request) + + log.debug('sending %s %s', method, path) + yield from self._co_send(buf) + if not self._out_remaining or expect100: + self._pending_requests.append((method, path, pending_body_size)) + + def _co_send(self, buf): + '''Send *buf* to server''' + + log.debug('trying to send %d bytes', len(buf)) + + if not isinstance(buf, memoryview): + buf = memoryview(buf) + + while True: + try: + if self._sock is None: + raise ConnectionClosed('connection has been closed locally') + len_ = self._sock.send(buf) + # An SSL socket has the nasty habit of returning zero + # instead of raising an exception when in non-blocking + # mode. + if len_ == 0: + raise BlockingIOError() + except (socket.timeout, ssl.SSLWantWriteError, BlockingIOError): + log.debug('yielding') + yield PollNeeded(self._sock.fileno(), POLLOUT) + continue + except (BrokenPipeError, ConnectionResetError): + raise ConnectionClosed('connection was interrupted') + except OSError as exc: + if exc.errno == errno.EINVAL: + # Blackhole routing, according to ip(7) + raise ConnectionClosed('ip route goes into black hole') + else: + raise + except InterruptedError: + log.debug('interrupted') + # According to send(2), this means that no data has been sent + # at all before the interruption, so we just try again. + continue + + log.debug('sent %d bytes', len_) + buf = buf[len_:] + if len(buf) == 0: + log.debug('done') + return + + def write(self, buf): + '''placeholder, will be replaced dynamically''' + eval_coroutine(self.co_write(buf), self.timeout) + + def co_write(self, buf): + '''Write request body data + + `ExcessBodyData` will be raised when attempting to send more data than + required to complete the request body of the active request. + ''' + + log.debug('start (len=%d)', len(buf)) + + if not self._out_remaining: + raise StateError('No active request with pending body data') + + (method, path, remaining) = self._out_remaining + if remaining is WAITING_FOR_100c: + raise StateError("can't write when waiting for 100-continue") + + if len(buf) > remaining: + raise ExcessBodyData('trying to write %d bytes, but only %d bytes pending' + % (len(buf), remaining)) + + try: + yield from self._co_send(buf) + except ConnectionClosed: + # If the server closed the connection, we pretend that all data + # has been sent, so that we can still read a (buffered) error + # response. + self._out_remaining = None + self._pending_requests.append((method, path, None)) + raise + + len_ = len(buf) + if len_ == remaining: + log.debug('body sent fully') + self._out_remaining = None + self._pending_requests.append((method, path, None)) + else: + self._out_remaining = (method, path, remaining - len_) + + log.debug('done') + + def response_pending(self): + '''Return `True` if there are still outstanding responses + + This includes responses that have been partially read. + ''' + + return self._sock is not None and len(self._pending_requests) > 0 + + def read_response(self): + '''placeholder, will be replaced dynamically''' + return eval_coroutine(self.co_read_response(), self.timeout) + + def co_read_response(self): + '''Read response status line and headers + + Return a `HTTPResponse` instance containing information about response + status, reason, and headers. The response body data must be retrieved + separately (e.g. using `.read` or `.readall`). + ''' + + log.debug('start') + + if len(self._pending_requests) == 0: + raise StateError('No pending requests') + + if self._in_remaining is not None: + raise StateError('Previous response not read completely') + + (method, path, body_size) = self._pending_requests[0] + + # Need to loop to handle any 1xx responses + while True: + (status, reason) = yield from self._co_read_status() + log.debug('got %03d %s', status, reason) + + hstring = yield from self._co_read_header() + header = email.message_from_string(hstring, policy=email.policy.HTTP) + + if status < 100 or status > 199: + break + + # We are waiting for 100-continue + if body_size is not None and status == 100: + break + + # Handle (expected) 100-continue + if status == 100: + assert self._out_remaining == (method, path, WAITING_FOR_100c) + + # We're ready to sent request body now + self._out_remaining = self._pending_requests.popleft() + self._in_remaining = None + + # Return early, because we don't have to prepare + # for reading the response body at this time + return HTTPResponse(method, path, status, reason, header, length=0) + + # Handle non-100 status when waiting for 100-continue + elif body_size is not None: + assert self._out_remaining == (method, path, WAITING_FOR_100c) + # RFC 2616 actually states that the server MAY continue to read + # the request body after it has sent a final status code + # (http://tools.ietf.org/html/rfc2616#section-8.2.3). However, + # that totally defeats the purpose of 100-continue, so we hope + # that the server behaves sanely and does not attempt to read + # the body of a request it has already handled. (As a side note, + # this ambuigity in the RFC also totally breaks HTTP pipelining, + # as we can never be sure if the server is going to expect the + # request or some request body data). + self._out_remaining = None + + # + # Prepare to read body + # + body_length = self._setup_read(method, status, header) + + # Don't require calls to co_read() et al if there is + # nothing to be read. + if self._in_remaining is None: + self._pending_requests.popleft() + + log.debug('done (in_remaining=%s)', self._in_remaining) + return HTTPResponse(method, path, status, reason, header, body_length) + + def _setup_read(self, method, status, header): + '''Prepare for reading response body + + Sets up `._encoding`, `_in_remaining` and returns Content-Length + (if available). + + See RFC 2616, sec. 4.4 for specific rules. + ''' + + # On error, the exception is stored in _encoding and raised on + # the next call to co_read() et al - that way we can still + # return the http status and headers. + + will_close = header.get('Connection', 'keep-alive').lower() == 'close' + + body_length = header['Content-Length'] + if body_length is not None: + try: + body_length = int(body_length) + except ValueError: + self._encoding = InvalidResponse('Invalid content-length: %s' + % body_length) + self._in_remaining = RESPONSE_BODY_ERROR + return None + + if (status == NO_CONTENT or status == NOT_MODIFIED or + 100 <= status < 200 or method == 'HEAD'): + log.debug('no content by RFC') + self._in_remaining = None + self._encoding = None + return 0 + + tc = header.get('Transfer-Encoding', 'identity').lower() + if tc == 'chunked': + log.debug('Chunked encoding detected') + self._encoding = Encodings.CHUNKED + self._in_remaining = 0 + return None + + elif tc != 'identity': + log.warning('Server uses invalid response encoding "%s"', tc) + self._encoding = InvalidResponse('Cannot handle %s encoding' % tc) + self._in_remaining = RESPONSE_BODY_ERROR + return None + + log.debug('identity encoding detected') + self._encoding = Encodings.IDENTITY + + if body_length is not None: + log.debug('Will read response body of %d bytes', body_length) + self._in_remaining = body_length or None + return body_length + + if will_close: + log.debug('no content-length, will read until EOF') + self._in_remaining = READ_UNTIL_EOF + return None + + log.debug('no content length and no chunked encoding, will raise on read') + self._encoding = UnsupportedResponse('No content-length and no chunked encoding') + self._in_remaining = RESPONSE_BODY_ERROR + return None + + def _co_read_status(self): + '''Read response line''' + + log.debug('start') + + # read status + try: + line = yield from self._co_readstr_until(b'\r\n', MAX_LINE_SIZE) + except _ChunkTooLong: + raise InvalidResponse('server send ridicously long status line') + + try: + version, status, reason = line.split(None, 2) + except ValueError: + try: + version, status = line.split(None, 1) + reason = "" + except ValueError: + # empty version will cause next test to fail. + version = "" + + if not version.startswith("HTTP/1"): + raise UnsupportedResponse('%s not supported' % version) + + # The status code is a three-digit number + try: + status = int(status) + if status < 100 or status > 999: + raise InvalidResponse('%d is not a valid status' % status) + except ValueError: + raise InvalidResponse('%s is not a valid status' % status) + + log.debug('done') + return (status, reason.strip()) + + def _co_read_header(self): + '''Read response header''' + + log.debug('start') + + # Peek into buffer. If the first characters are \r\n, then the header + # is empty (so our search for \r\n\r\n would fail) + rbuf = self._rbuf + if len(rbuf) < 2: + yield from self._co_fill_buffer(2) + if rbuf.d[rbuf.b:rbuf.b+2] == b'\r\n': + log.debug('done (empty header)') + rbuf.b += 2 + return '' + + try: + hstring = yield from self._co_readstr_until(b'\r\n\r\n', MAX_HEADER_SIZE) + except _ChunkTooLong: + raise InvalidResponse('server sent ridicously long header') + + log.debug('done (%d characters)', len(hstring)) + return hstring + + def read(self, len_=None): + '''placeholder, will be replaced dynamically''' + if len_ is None: + return self.readall() + buf = eval_coroutine(self.co_read(len_), self.timeout) + + # Some modules like TextIOWrapper unfortunately rely on read() + # to return bytes, and do not accept bytearrays or memoryviews. + # cf. http://bugs.python.org/issue21057 + if sys.version_info < (3,5,0) and not isinstance(buf, bytes): + buf = bytes(buf) + return buf + + def co_read(self, len_=None): + '''Read up to *len_* bytes of response body data + + This method may return less than *len_* bytes, but will return ``b''`` only + if the response body has been read completely. + + If *len_* is `None`, this method returns the entire response body. + ''' + + log.debug('start (len=%d)', len_) + + if self._in_remaining is RESPONSE_BODY_ERROR: + raise self._encoding + elif len_ is None: + return (yield from self.co_readall()) + elif len_ == 0 or self._in_remaining is None: + return b'' + elif self._encoding is Encodings.IDENTITY: + return (yield from self._co_read_id(len_)) + elif self._encoding is Encodings.CHUNKED: + return (yield from self._co_read_chunked(len_=len_)) + else: + raise RuntimeError('ooops, this should not be possible') + + def read_raw(self, size): + '''Read *size* bytes of uninterpreted data + + This method may be used even after `UnsupportedResponse` or + `InvalidResponse` has been raised. It reads raw data from the socket + without attempting to interpret it. This is probably only useful for + debugging purposes to take a look at the raw data received from the + server. This method blocks if no data is available, and returns ``b''`` + if the connection has been closed. + + Calling this method will break the internal state and switch the socket + to blocking operation. The connection has to be closed and reestablished + afterwards. + + **Don't use this method unless you know exactly what you are doing**. + ''' + + if self._sock is None: + raise ConnectionClosed('connection has been closed locally') + + self._sock.setblocking(True) + + buf = bytearray() + rbuf = self._rbuf + while len(buf) < size: + len_ = min(size - len(buf), len(rbuf)) + if len_ < len(rbuf): + buf += rbuf.d[rbuf.b:rbuf.b+len_] + rbuf.b += len_ + elif len_ == 0: + buf2 = self._sock.recv(size - len(buf)) + if not buf2: + break + buf += buf2 + else: + buf += rbuf.exhaust() + + return buf + + def readinto(self, buf): + '''placeholder, will be replaced dynamically''' + return eval_coroutine(self.co_readinto(buf), self.timeout) + + def co_readinto(self, buf): + '''Read response body data into *buf* + + Return the number of bytes written or zero if the response body has been + read completely. + + *buf* must implement the memoryview protocol. + ''' + + log.debug('start (buflen=%d)', len(buf)) + + if self._in_remaining is RESPONSE_BODY_ERROR: + raise self._encoding + elif len(buf) == 0 or self._in_remaining is None: + return 0 + elif self._encoding is Encodings.IDENTITY: + return (yield from self._co_readinto_id(buf)) + elif self._encoding is Encodings.CHUNKED: + return (yield from self._co_read_chunked(buf=buf)) + else: + raise RuntimeError('ooops, this should not be possible') + + def _co_read_id(self, len_): + '''Read up to *len* bytes of response body assuming identity encoding''' + + log.debug('start (len=%d)', len_) + assert self._in_remaining is not None + + if not self._in_remaining: + # Body retrieved completely, clean up + self._in_remaining = None + self._pending_requests.popleft() + return b'' + + rbuf = self._rbuf + if self._in_remaining is not READ_UNTIL_EOF: + len_ = min(len_, self._in_remaining) + log.debug('updated len_=%d', len_) + + # If buffer is empty, reset so that we start filling from + # beginning. This check is already done by _try_fill_buffer(), but we + # have to do it here or we never enter the while loop if the buffer + # is empty but has no capacity (rbuf.b == rbuf.e == len(rbuf.d)) + if rbuf.b == rbuf.e: + rbuf.b = 0 + rbuf.e = 0 + + # Loop while we could return more data than we have buffered + # and buffer is not full + while len(rbuf) < len_ and rbuf.e < len(rbuf.d): + got_data = self._try_fill_buffer() + if got_data is None: + if rbuf: + log.debug('nothing more to read') + break + else: + log.debug('buffer empty and nothing to read, yielding..') + yield PollNeeded(self._sock.fileno(), POLLIN) + elif got_data == 0: + if self._in_remaining is READ_UNTIL_EOF: + log.debug('connection closed, %d bytes in buffer', len(rbuf)) + self._in_remaining = len(rbuf) + break + else: + raise ConnectionClosed('server closed connection') + + len_ = min(len_, len(rbuf)) + if self._in_remaining is not READ_UNTIL_EOF: + self._in_remaining -= len_ + + if len_ < len(rbuf): + buf = rbuf.d[rbuf.b:rbuf.b+len_] + rbuf.b += len_ + else: + buf = rbuf.exhaust() + + # When reading until EOF, it is possible that we read only the EOF. In + # this case we won't get called again, so we need to clean-up the + # request. This can not happen when we know the number of remaining + # bytes, because in this case either the check at the start of the + # function hits, or we can't read the remaining data and raise. + if len(buf) == 0: + assert self._in_remaining == 0 + self._in_remaining = None + self._pending_requests.popleft() + + log.debug('done (%d bytes)', len(buf)) + return buf + + def _co_readinto_id(self, buf): + '''Read response body into *buf* assuming identity encoding''' + + log.debug('start (buflen=%d)', len(buf)) + + assert self._in_remaining is not None + if not self._in_remaining: + # Body retrieved completely, clean up + self._in_remaining = None + self._pending_requests.popleft() + return 0 + + rbuf = self._rbuf + if not isinstance(buf, memoryview): + buf = memoryview(buf) + if self._in_remaining is READ_UNTIL_EOF: + len_ = len(buf) + else: + len_ = min(len(buf), self._in_remaining) + log.debug('set len_=%d', len_) + + # First use read buffer contents + pos = min(len(rbuf), len_) + if pos: + log.debug('using buffered data') + buf[:pos] = rbuf.d[rbuf.b:rbuf.b+pos] + rbuf.b += pos + if self._in_remaining is not READ_UNTIL_EOF: + self._in_remaining -= pos + + # If we've read enough, return immediately + if pos == len_: + log.debug('done (buffer filled completely)') + return pos + + # Otherwise, prepare to read more from socket + log.debug('got %d bytes from buffer', pos) + assert not len(rbuf) + + while True: + log.debug('trying to read from socket') + if self._sock is None: + raise ConnectionClosed('connection has been closed locally') + try: + read = self._sock.recv_into(buf[pos:len_]) + except (ConnectionResetError, BrokenPipeError): + raise ConnectionClosed('connection was interrupted') + except (socket.timeout, ssl.SSLWantReadError, BlockingIOError): + if pos: + log.debug('done, no additional data available') + return pos + else: + log.debug('no data yet and nothing to read, yielding..') + yield PollNeeded(self._sock.fileno(), POLLIN) + continue + + if not read: + if self._in_remaining is READ_UNTIL_EOF: + log.debug('reached EOF') + self._in_remaining = 0 + return pos + else: + raise ConnectionClosed('server closed connection') + log.debug('got %d bytes from socket', read) + + if self._in_remaining is not READ_UNTIL_EOF: + self._in_remaining -= read + pos += read + if pos == len_: + log.debug('done (buffer filled completely)') + return pos + + def _co_read_chunked(self, len_=None, buf=None): + '''Read response body assuming chunked encoding + + If *len_* is not `None`, reads up to *len_* bytes of data and returns + a `bytes-like object`. If *buf* is not `None`, reads data into *buf*. + ''' + + log.debug('start (%s mode)', 'readinto' if buf else 'read') + assert (len_ is None) != (buf is None) + assert bool(len_) or bool(buf) + assert isinstance(self._in_remaining, int) + + if self._in_remaining == 0: + log.debug('starting next chunk') + try: + line = yield from self._co_readstr_until(b'\r\n', MAX_LINE_SIZE) + except _ChunkTooLong: + raise InvalidResponse('could not find next chunk marker') + + i = line.find(";") + if i >= 0: + log.debug('stripping chunk extensions: %s', line[i:]) + line = line[:i] # strip chunk-extensions + try: + self._in_remaining = int(line, 16) + except ValueError: + raise InvalidResponse('Cannot read chunk size %r' % line[:20]) + + log.debug('chunk size is %d', self._in_remaining) + if self._in_remaining == 0: + self._in_remaining = None + self._pending_requests.popleft() + + if self._in_remaining is None: + res = 0 if buf else b'' + elif buf: + res = yield from self._co_readinto_id(buf) + else: + res = yield from self._co_read_id(len_) + + if not self._in_remaining: + log.debug('chunk complete') + yield from self._co_read_header() + + log.debug('done') + return res + + def _co_readstr_until(self, substr, maxsize): + '''Read from server until *substr*, and decode to latin1 + + If *substr* cannot be found in the next *maxsize* bytes, + raises `_ChunkTooLong`. + ''' + + if not isinstance(substr, (bytes, bytearray, memoryview)): + raise TypeError('*substr* must be bytes-like') + + log.debug('reading until %s', substr) + + rbuf = self._rbuf + sub_len = len(substr) + + # Make sure that substr cannot be split over more than one part + assert len(rbuf.d) > sub_len + + parts = [] + while True: + # substr may be split between last part and current buffer + # This isn't very performant, but it should be pretty rare + if parts and sub_len > 1: + buf = _join((parts[-1][-sub_len:], + rbuf.d[rbuf.b:min(rbuf.e, rbuf.b+sub_len-1)])) + idx = buf.find(substr) + if idx >= 0: + idx -= sub_len + break + + #log.debug('rbuf is: %s', rbuf.d[rbuf.b:min(rbuf.e, rbuf.b+512)]) + stop = min(rbuf.e, rbuf.b + maxsize) + idx = rbuf.d.find(substr, rbuf.b, stop) + + if idx >= 0: # found + break + if stop != rbuf.e: + raise _ChunkTooLong() + + # If buffer is full, store away the part that we need for sure + if rbuf.e == len(rbuf.d): + log.debug('buffer is full, storing part') + buf = rbuf.exhaust() + parts.append(buf) + maxsize -= len(buf) + + # Refill buffer + while True: + res = self._try_fill_buffer() + if res is None: + log.debug('need more data, yielding') + yield PollNeeded(self._sock.fileno(), POLLIN) + elif res == 0: + raise ConnectionClosed('server closed connection') + else: + break + + log.debug('found substr at %d', idx) + idx += len(substr) + buf = rbuf.d[rbuf.b:idx] + rbuf.b = idx + + if parts: + parts.append(buf) + buf = _join(parts) + + try: + return buf.decode('latin1') + except UnicodeDecodeError: + raise InvalidResponse('server response cannot be decoded to latin1') + + def _try_fill_buffer(self): + '''Try to fill up read buffer + + Returns the number of bytes read into buffer, or `None` if no data was + available on the socket. Return zero if the TCP connection has been + properly closed but the socket object still exists. On other problems + (e.g. if the socket object has been destroyed or the connection + interrupted), raise `ConnectionClosed`. + ''' + + log.debug('start') + rbuf = self._rbuf + + # If buffer is empty, reset so that we start filling from beginning + if rbuf.b == rbuf.e: + rbuf.b = 0 + rbuf.e = 0 + + # There should be free capacity + assert rbuf.e < len(rbuf.d) + + if self._sock is None: + raise ConnectionClosed('connection has been closed locally') + + try: + len_ = self._sock.recv_into(memoryview(rbuf.d)[rbuf.e:]) + except (socket.timeout, ssl.SSLWantReadError, BlockingIOError): + log.debug('done (nothing ready)') + return None + except (ConnectionResetError, BrokenPipeError): + raise ConnectionClosed('connection was interrupted') + + rbuf.e += len_ + log.debug('done (got %d bytes)', len_) + return len_ + + def _co_fill_buffer(self, len_): + '''Make sure that there are at least *len_* bytes in buffer''' + + rbuf = self._rbuf + if len_ > len(rbuf.d): + raise ValueError('Requested more bytes than buffer has capacity') + while len(rbuf) < len_: + if len(rbuf.d) - rbuf.b < len_: + self._rbuf.compact() + res = self._try_fill_buffer() + if res is None: + yield PollNeeded(self._sock.fileno(), POLLIN) + elif res == 0: + raise ConnectionClosed('server closed connection') + + def readall(self): + '''placeholder, will be replaced dynamically''' + return eval_coroutine(self.co_readall(), self.timeout) + + def co_readall(self): + '''Read and return complete response body''' + + if self._in_remaining is None: + return b'' + + log.debug('start') + parts = [] + while True: + buf = yield from self.co_read(BUFFER_SIZE) + log.debug('got %d bytes', len(buf)) + if not buf: + break + parts.append(buf) + buf = _join(parts) + log.debug('done (%d bytes)', len(buf)) + return buf + + def discard(self): + '''placeholder, will be replaced dynamically''' + return eval_coroutine(self.co_discard(), self.timeout) + + def co_discard(self): + '''Read and discard current response body''' + + if self._in_remaining is None: + return + + log.debug('start') + buf = memoryview(bytearray(BUFFER_SIZE)) + while True: + len_ = yield from self.co_readinto(buf) + if not len_: + break + log.debug('discarding %d bytes', len_) + log.debug('done') + + def reset(self): + '''Reset HTTP connection + + This method resets the status of the HTTP connection after an exception + has occured. Any cached data and pending responses are discarded. + ''' + self.disconnect() + + def disconnect(self): + '''Close HTTP connection''' + + log.debug('start') + if self._sock: + try: + self._sock.shutdown(socket.SHUT_RDWR) + except OSError: + # When called to reset after connection problems, socket + # may have shut down already. + pass + self._sock.close() + self._sock = None + self._rbuf.clear() + else: + log.debug('already closed') + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.disconnect() + return False + +def _extend_HTTPConnection_docstrings(): + + co_suffix = '\n\n' + textwrap.fill( + 'This method returns a coroutine. `.%s` is a regular method ' + 'implementing the same functionality.', width=78) + reg_suffix = '\n\n' + textwrap.fill( + 'This method may block. `.co_%s` provides a coroutine ' + 'implementing the same functionality without blocking.', width=78) + + for name in ('read', 'read_response', 'readall', 'readinto', 'send_request', + 'write', 'discard'): + fn = getattr(HTTPConnection, name) + cofn = getattr(HTTPConnection, 'co_' + name) + + fn.__doc__ = getdoc(cofn) + reg_suffix % name + cofn.__doc__ = getdoc(cofn) + co_suffix % name + +_extend_HTTPConnection_docstrings() + +if sys.version_info < (3,4,0): + def _join(parts): + '''Join a sequence of byte-like objects + + This method is necessary because `bytes.join` does not work with + memoryviews prior to Python 3.4. + ''' + + size = 0 + for part in parts: + size += len(part) + + buf = bytearray(size) + i = 0 + for part in parts: + len_ = len(part) + buf[i:i+len_] = part + i += len_ + + return buf +else: + def _join(parts): + return b''.join(parts) + +def eval_coroutine(crt, timeout=None): + '''Evaluate *crt* (polling as needed) and return its result + + If *timeout* seconds pass without being able to send or receive + anything, raises `ConnectionTimedOut`. + ''' + + try: + while True: + log.debug('polling') + if not next(crt).poll(timeout=timeout): + raise ConnectionTimedOut() + except StopIteration as exc: + return exc.value + +def create_socket(address): + '''Create socket connected to *address** + + Use `socket.create_connection` to create a connected socket and return + it. If a DNS related exception is raised, capture it and attempt to + determine if the dns server is not reachable, or if the host name could + not be resolved. Then raise either `NoSuchAddress` or + `NameResolutionError` as appropriate. + + To distinguish between an unresolvable hostname and a problem with the + DNS server, attempt to resolve the addresses in `DNS_TEST_HOSTNAMES`. If + at least one test hostname can be resolved, assume that the DNS server + is available and that *address* can not be resolved. + ''' + + try: + return socket.create_connection(address) + + # The exception unfortunately does not help us to distinguish between + # permanent and temporary problems. See: + # https://stackoverflow.com/questions/24855168/ + # https://stackoverflow.com/questions/24855669/ + except (socket.gaierror, socket.herror) as exc: + if exc.errno not in (socket.EAI_AGAIN, socket.EAI_NONAME): + raise + + # Try to resolve test hosts + for (hostname, port) in DNS_TEST_HOSTNAMES: + try: + socket.getaddrinfo(hostname, port) + except (socket.gaierror, socket.herror) as exc: + if exc.errno not in (socket.EAI_AGAIN, socket.EAI_NONAME): + raise + # Not reachable, try next one + else: + # Reachable, now try to resolve original address + # again (maybe dns was only down briefly) + break + else: + # No host was reachable + raise DNSUnavailable(address[0]) + + # Try to connect to original host again + try: + return socket.create_connection(address) + except (socket.gaierror, socket.herror) as exc: + if exc.errno not in (socket.EAI_AGAIN, socket.EAI_NONAME): + raise + raise HostnameNotResolvable(address[0]) + + +def is_temp_network_error(exc): + '''Return true if *exc* represents a potentially temporary network problem + + DNS resolution errors (`socket.gaierror` or `socket.herror`) are considered + permanent, because `HTTPConnection` employs a heuristic to convert these + exceptions to `HostnameNotResolvable` or `DNSUnavailable` instead. + ''' + + if isinstance(exc, (socket.timeout, ConnectionError, TimeoutError, InterruptedError, + ConnectionClosed, ssl.SSLZeroReturnError, ssl.SSLEOFError, + ssl.SSLSyscallError, ConnectionTimedOut, DNSUnavailable)): + return True + + elif isinstance(exc, OSError): + # We have to be careful when retrieving errno codes, because + # not all of them may exist on every platform. + for errcode in ('EHOSTDOWN', 'EHOSTUNREACH', 'ENETDOWN', + 'ENETRESET', 'ENETUNREACH', 'ENOLINK', + 'ENONET', 'ENOTCONN', 'ENXIO', 'EPIPE', + 'EREMCHG', 'ESHUTDOWN', 'ETIMEDOUT'): + try: + if getattr(errno, errcode) == exc.errno: + return True + except AttributeError: + pass + + return False + + +class CaseInsensitiveDict(MutableMapping): + """A case-insensitive `dict`-like object. + + Implements all methods and operations of + :class:`collections.abc.MutableMapping` as well as `.copy`. + + All keys are expected to be strings. The structure remembers the case of the + last key to be set, and :meth:`!iter`, :meth:`!keys` and :meth:`!items` will + contain case-sensitive keys. However, querying and contains testing is case + insensitive:: + + cid = CaseInsensitiveDict() + cid['Accept'] = 'application/json' + cid['aCCEPT'] == 'application/json' # True + list(cid) == ['Accept'] # True + + For example, ``headers['content-encoding']`` will return the value of a + ``'Content-Encoding'`` response header, regardless of how the header name + was originally stored. + + If the constructor, :meth:`!update`, or equality comparison operations are + given multiple keys that have equal lower-case representions, the behavior + is undefined. + """ + + def __init__(self, data=None, **kwargs): + self._store = dict() + if data is None: + data = {} + self.update(data, **kwargs) + + def __setitem__(self, key, value): + # Use the lowercased key for lookups, but store the actual + # key alongside the value. + self._store[key.lower()] = (key, value) + + def __getitem__(self, key): + return self._store[key.lower()][1] + + def __delitem__(self, key): + del self._store[key.lower()] + + def __iter__(self): + return (casedkey for casedkey, mappedvalue in self._store.values()) + + def __len__(self): + return len(self._store) + + def lower_items(self): + """Like :meth:`!items`, but with all lowercase keys.""" + return ( + (lowerkey, keyval[1]) + for (lowerkey, keyval) + in self._store.items() + ) + + def __eq__(self, other): + if isinstance(other, Mapping): + other = CaseInsensitiveDict(other) + else: + return NotImplemented + # Compare insensitively + return dict(self.lower_items()) == dict(other.lower_items()) + + # Copy is required + def copy(self): + return CaseInsensitiveDict(self._store.values()) + + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, dict(self.items())) + + +if asyncio: + class AioFuture(asyncio.Future): + ''' + This class wraps a coroutine that yields `PollNeeded` instances + into an `asyncio` compatible `~asyncio.Future`. + + This is done by registering a callback with the event loop that resumes + the coroutine when the requested IO is available. + ''' + + #: Set of fds that that any `_Future` instance currently has + #: read callbacks registered for (class attribute) + _read_fds = dict() + + #: Set of fds that that any `_Future` instance currently has + #: write callbacks registered for (class attribute) + _write_fds = dict() + + def __init__(self, crt, loop=None): + super().__init__(loop=loop) + self._crt = crt + + #: The currently pending io request (that we have registered + #: callbacks for). + self._io_req = None + + self._loop.call_soon(self._resume_crt) + + def _resume_crt(self, exc=None): + '''Resume coroutine + + If coroutine has completed, mark self as done. Otherwise, reschedule + call when requested io is available. If *exc* is specified, raise + *exc* in coroutine. + ''' + + log.debug('start') + try: + if exc is not None: + io_req = self._crt.throw(exc) + else: + io_req = next(self._crt) + except Exception as exc: + if isinstance(exc, StopIteration): + log.debug('coroutine completed') + self.set_result(exc.value) + else: + log.debug('coroutine raised exception') + self.set_exception(exc) + io_req = self._io_req + if io_req: + # This is a bit fragile.. what if there is more than one + # reader or writer? However, in practice this should not be + # the case: they would read or write unpredictable parts of + # the input/output. + if io_req.mask & POLLIN: + self._loop.remove_reader(io_req.fd) + del self._read_fds[io_req.fd] + if io_req.mask & POLLOUT: + self._loop.remove_writer(io_req.fd) + del self._write_fds[io_req.fd] + self._io_req = None + return + + if not isinstance(io_req, PollNeeded): + self._loop.call_soon(self._resume_crt, + TypeError('Coroutine passed to asyncio_future did not yield ' + 'PollNeeded instance!')) + return + + if io_req.mask & POLLIN: + reader = self._read_fds.get(io_req.fd, None) + if reader is None: + log.debug('got poll needed, registering reader') + self._loop.add_reader(io_req.fd, self._resume_crt) + self._read_fds[io_req.fd] = self + elif reader is self: + log.debug('got poll needed, reusing read callback') + else: + self._loop.call_soon(self._resume_crt, + RuntimeError('There is already a read callback for this socket')) + return + + if io_req.mask & POLLOUT: + writer = self._read_fds.get(io_req.fd, None) + if writer is None: + log.debug('got poll needed, registering writer') + self._loop.add_writer(io_req.fd, self._resume_crt) + self._write_fds[io_req.fd] = self + elif writer is self: + log.debug('got poll needed, reusing write callback') + else: + self._loop.call_soon(self._resume_crt, + RuntimeError('There is already a write callback for this socket')) + return + + self._io_req = io_req diff --git a/examples/extract_links.py b/examples/extract_links.py new file mode 100755 index 0000000..8f51149 --- /dev/null +++ b/examples/extract_links.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +''' +Extract all links from a URL. +''' + +import sys +import os.path +from io import TextIOWrapper +from html.parser import HTMLParser +from urllib.parse import urlsplit, urljoin, urlunsplit +import re +import ssl + +# We are running from the dugong source directory, append it to module path so +# that we can fallback on it if dugong hasn't been installed yet. +if __name__ == '__main__': + basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +else: + basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'dugong', '__init__.py'))): + sys.path.append(basedir) + +from dugong import HTTPConnection + +# When running from HG repo, enable all warnings +if os.path.exists(os.path.join(basedir, '.hg')): + import warnings + warnings.simplefilter('error') + +class LinkExtractor(HTMLParser): + def __init__(self): + if sys.version_info < (3,4): + # Python 3.3 doesn't know about convert_charrefs + super().__init__() + else: + super().__init__(convert_charrefs=True) + self.links = [] + + def handle_starttag(self, tag, attrs): + if tag != 'a': + return + + for (name, val) in attrs: + if name == 'href': + self.links.append(val) + break + +def main(): + if len(sys.argv) != 2: + raise SystemExit('Usage: %s ' % sys.argv[0]) + url = sys.argv[1] + url_els = urlsplit(url) + + if url_els.scheme == 'https': + ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ssl_context.options |= ssl.OP_NO_SSLv2 + ssl_context.verify_mode = ssl.CERT_REQUIRED + ssl_context.set_default_verify_paths() + else: + ssl_context = None + + with HTTPConnection(url_els.hostname, port=url_els.port, + ssl_context=ssl_context) as conn: + path = urlunsplit(('', '') + url_els[2:4] + ('',)) or '/' + conn.send_request('GET', path) + resp = conn.read_response() + if resp.status != 200: + raise SystemExit('%d %s' % (resp.status, resp.reason)) + + # Determine if we're reading text or binary data, and (in case of text), + # what character set is being used. + if 'Content-Type' not in resp.headers: + type_ = 'application/octet-stream' + else: + type_ = resp.headers['Content-Type'] + + hit = re.match(r'text/x?html(?:; charset=(.+))?$', type_) + if not hit: + raise SystemExit('Server did not send html but %s' % type_) + + if hit.group(1): + charset = hit.group(1) + else: + charset = 'latin1' + + html_stream = TextIOWrapper(conn, encoding=charset) + parser = LinkExtractor() + + while True: + buf = html_stream.read(16*1024) + if not buf: + break + parser.feed(buf) + + for link in parser.links: + print(urljoin(url, link)) + +if __name__ == '__main__': + main() diff --git a/examples/httpcat.py b/examples/httpcat.py new file mode 100755 index 0000000..e517d66 --- /dev/null +++ b/examples/httpcat.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +''' +Retrieve a list of URLs and print them to stdout. +''' + +import sys +import os.path +from io import TextIOWrapper +import re +from urllib.parse import urlsplit + +# We are running from the dugong source directory, append it to module path so +# that we can fallback on it if dugong hasn't been installed yet. +if __name__ == '__main__': + basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +else: + basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'dugong', '__init__.py'))): + sys.path.append(basedir) + +# When running from HG repo, enable all warnings +if os.path.exists(os.path.join(basedir, '.hg')): + import warnings + warnings.simplefilter('error') + +from dugong import HTTPConnection, BUFFER_SIZE + +for arg in sys.argv[1:]: + url = urlsplit(arg) + assert url.scheme == 'http' + path = url.path + if url.query: + path += '?' + url.query + + with HTTPConnection(url.hostname, url.port) as conn: + conn.send_request('GET', path) + resp = conn.read_response() + if resp.status != 200: + raise SystemExit('%d %s' % (resp.status, resp.reason)) + + # Determine if we're reading text or binary data, and (in case of text), + # what character set is being used. + if 'Content-Type' not in resp.headers: + type_ = 'application/octet-stream' + else: + type_ = resp.headers['Content-Type'] + + hit = re.match(r'(.+?)(?:; charset=(.+))?$', type_) + if not hit: + raise SystemExit('Unable to parse content-type: %s' % type_) + if hit.group(2): + charset = hit.group(2) + elif hit.group(1).startswith('text/'): + charset = 'latin1' + else: + charset = None # binary data + + if charset: + instream = TextIOWrapper(conn, encoding=charset) + outstream = sys.stdout + else: + instream = conn + # Since we're writing bytes rather than text, we need to bypass + # any encoding. + outstream = sys.stdout.raw + + while True: + buf = instream.read(BUFFER_SIZE) + if not buf: + break + outstream.write(buf) diff --git a/examples/pipeline1.py b/examples/pipeline1.py new file mode 100755 index 0000000..83ee664 --- /dev/null +++ b/examples/pipeline1.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +import sys +import os.path +from urllib.parse import urlsplit, urlunsplit + +# We are running from the dugong source directory, append it to module path so +# that we can fallback on it if dugong hasn't been installed yet. +if __name__ == '__main__': + basedir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..')) +else: + basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) +if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'dugong', '__init__.py'))): + sys.path.append(basedir) + +# When running from HG repo, enable all warnings +if os.path.exists(os.path.join(basedir, '.hg')): + import warnings + warnings.simplefilter('error') + +# Assemble path list +hostname = None +path_list = [] +for url in sys.argv[1:]: + o = urlsplit(url) + if hostname is None: + hostname = o.hostname + port = o.port + elif (hostname, port) != (o.hostname, o.port): + raise SystemExit('Can only pipeline to one host') + if o.scheme != 'http': + raise SystemExit('Can only do http') + path_list.append(urlunsplit(('', '') + o[2:4] + ('',))) + + +# Code from here on is included in documentation +# start-example +import asyncio +import atexit +from dugong import HTTPConnection, AioFuture + +# Get a MainLoop instance from the asyncio module to switch +# between coroutines (and clean up at program exit) +loop = asyncio.get_event_loop() +atexit.register(loop.close) + +with HTTPConnection(hostname, port) as conn: + # This generator function returns a coroutine that sends + # all the requests. + def send_requests(): + for path in path_list: + yield from conn.co_send_request('GET', path) + + # This generator function returns a coroutine that reads + # all the responses + def read_responses(): + bodies = [] + for path in path_list: + resp = yield from conn.co_read_response() + assert resp.status == 200 + buf = yield from conn.co_readall() + bodies.append(buf) + return bodies + + # Create the coroutines + send_crt = send_requests() + recv_crt = read_responses() + + # Register the coroutines with the event loop + send_future = AioFuture(send_crt, loop=loop) + recv_future = AioFuture(recv_crt, loop=loop) + + # Run the event loop until the receive coroutine is done (which + # implies that all the requests must have been sent as well): + loop.run_until_complete(recv_future) + + # Get the result returned by the coroutine + bodies = recv_future.result() + +# end-example diff --git a/rst/_templates/localtoc.html b/rst/_templates/localtoc.html new file mode 100644 index 0000000..bf18c4c --- /dev/null +++ b/rst/_templates/localtoc.html @@ -0,0 +1,2 @@ +

{{ _('Table Of Contents') }}

+{{ toctree() }} diff --git a/rst/api.rst b/rst/api.rst new file mode 100644 index 0000000..f90638d --- /dev/null +++ b/rst/api.rst @@ -0,0 +1,101 @@ +API Reference +============= + +.. currentmodule:: dugong + +Classes +------- + +.. autoclass:: HTTPConnection + :members: + +.. autoclass:: HTTPResponse + :members: + +.. autoclass:: BodyFollowing + :members: + +.. autoclass:: CaseInsensitiveDict + :members: + +.. autoclass:: PollNeeded + :members: + +.. autoclass:: AioFuture + +Functions +--------- + +.. autofunction:: is_temp_network_error + + +Exceptions +---------- + +Dugong functions may pass through any exceptions raised by the +:ref:`socket ` and `ssl.SSLSocket` methods. In +addition to that, the following dugong-specific exceptions may be +raised as well: + +.. autoexception:: ConnectionClosed + :members: + +.. autoexception:: InvalidResponse + :members: + +.. autoexception:: UnsupportedResponse + :members: + +.. autoexception:: ExcessBodyData + :members: + +.. autoexception:: StateError + :members: + +.. autoexception:: ConnectionTimedOut + :members: + +.. autoexception:: HostnameNotResolvable + :members: + +.. autoexception:: DNSUnavailable + :members: + +Constants +--------- + +.. autodata:: MAX_LINE_SIZE + +.. autodata:: MAX_HEADER_SIZE + +.. autodata:: DNS_TEST_HOSTNAMES + +Thread Safety +------------- + +Dugong is not generally threadsafe. However, simultaneous use of the +same `HTTPConnection` instance by two threads is supported if once +thread is restricted to sending requests, and the other thread +restricted to reading responses. + +Avoiding Deadlocks +------------------ + +The `HTTPConnection` class allows you to send an unlimited number of +requests to the server before reading any of the responses. However, at some +point the transmit and receive buffers on both the ends of the connection +will fill up, and no more requests can be send before at least some of the +responses are read, and attempts to send more data to the server will +block. If the thread that attempts to send data is is also responsible for +reading the responses, this will result in a deadlock. + +There are several ways to avoid this: + +- Do not send a new request before the last response has been read. This is + the easiest solution, but it means that no HTTP pipelining can be used. + +- Use different threads for sending requests and receiving responses. + +- Use the coroutine based API (see :ref:`coroutine_pipelining` in the + tutorial). + diff --git a/rst/conf.py b/rst/conf.py new file mode 100644 index 0000000..f312145 --- /dev/null +++ b/rst/conf.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +#pylint: disable-all +#@PydevCodeAnalysisIgnore + +import sys +import os.path + +sys.path.append(os.path.abspath('..')) + +extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx' ] +intersphinx_mapping = {'python': ('http://docs.python.org/3/', None) } +templates_path = ['_templates'] +source_suffix = '.rst' +source_encoding = 'utf-8' +master_doc = 'index' +nitpicky = True +project = u'Dugong' +copyright = u'2013-2014, Nikolaus Rath' +default_role = 'py:obj' +primary_domain = 'py' +add_module_names = False +autodoc_member_order = 'groupwise' +pygments_style = 'sphinx' +highlight_language = 'python' +html_theme = 'default' +html_use_modindex = False +html_use_index = True +html_split_index = False +html_show_sourcelink = False + diff --git a/rst/coroutines.rst b/rst/coroutines.rst new file mode 100644 index 0000000..45b7e81 --- /dev/null +++ b/rst/coroutines.rst @@ -0,0 +1,276 @@ +.. currentmodule:: dugong + +.. _coroutines: + +============= +Coroutine API +============= + +This section assumes some basic familiarity with coroutines. If you +don't know what they are, you are missing out a lot and should read up +on them right away (e.g. on `Wikipedia `_, `PEP +342`_, `PEP 380`_ and `dabeaz.com`_). + +To refresh your memory: coroutines in Python are generators, and are +obtained by calling generator functions (i.e, functions that use +``yield`` in their definiton). A coroutine can be resumed by passing +it to the built-in `next` function, or calling its `~generator.send` +method. A coroutine can pass the control flow back to the caller by +:ref:`yielding ` values using the ``yield`` +expression. When the coroutine eventually terminates, the last call to +`next` or `~generator.send` will raise a `StopIteration` exception, +whose *value* attribute holds the return value of the coroutine. A +coroutine *A* may also *yield from* another coroutine *B* using the +``yield from`` expression. In this case, the control flow will pass +between *A*'s caller and *B* until *B* terminates. When *B* has +terminated, its return value becomes the result of the ``yield from`` +expression in *A*, and execution continues in *A*. + +In Dugong, a method or function whose name begins with ``co_`` will +return a coroutine. These coroutines are non-blocking. Whenever they +need to perform an I/O operation that would block (ie., sending data +to the server or receiving data from the server), they yield a +`PollNeeded` instance instead, and expect to be resumed when the +operation can be carried out without blocking. + +The `PollNeeded` instance contains information about the I/O request +that the coroutine would like to perform. The `~PollNeeded.fd` +attribute is a file descriptor, and the `~PollNeeded.mask` attribute +is an :ref:`epoll ` compatible event mask. Therefore, a +very simple way to wait for a coroutine to complete is to use a +`~select.select` loop:: + + from select import select, POLLIN + + # establish connection, send request, read response header + + # Create coroutine + crt = conn.co_readall() + try: + while True: + # Resume coroutine + io_req = next(crt) + + # Coroutine has returned because I/O is not ready, + # prepare select call + read_fds = (io_req.fd,) if io_req.mask & POLLIN else () + write_fds = (io_req.fd,) if io_req.mask & POLLOUT else () + + # Wait for I/O readiness + select(read_fds, write_fds, ()) + except StopIteration as exc: + # Coroutine has completed, retrieve result + body = exc.value + +This loop is in fact fully equivalent to a simple :: + + body = conn.readall() + +so in this case there really wasn't much point in using a +coroutine. This is because coroutines really only make sense if you +have more than one active coroutine. However, in that case the +necessary loop construction becomes a lot more complicated. Luckily +enough, Dugong is compatible with the `asyncio` module, so you can use +the asyncio event loop to schedule your Dugong coroutines. + + +Using asyncio Event-Loops +========================= + +In order to schedule a Dugong coroutine in an asyncio event loop, you +have to create an `asyncio.Future` for the coroutine. This is done +with the `dugong.AioFuture` class (which inherits from +`asyncio.Future`). The reason for this additional wrapper is that the +asyncio event loop, even though very powerful, does not know how to +interpret the `PollNeeded` instances that are yielded by Dugong +coroutines. It would have been possible to have Dugong coroutines +yield `asyncio.Future` instances directly, but this would have meant +to introduce a hard dependency on asyncio, which was deemend +undesirable. + +Using asyncio, the above example becomes much simpler:: + + import asyncio + import atexit + + # establish connection, send request, read response header + + # Create coroutine + crt = conn.co_readall() + + # Get a MainLoop instance from the asyncio module to switch + # between the coroutines as needed + loop = asyncio.get_event_loop() + atexit.register(loop.close) + + # Create and schedule asyncio future + fut = AioFuture(crt, loop=loop) + + # Run the event loop + loop.run_until_complete(fut) + + # Get the result returned by the coroutine + body = fut.result() + +The generalization to multiple coroutines is now +straightforward. Suppose you want to retrieve a number of documents +from different servers. You could use threads, but this makes the +program hard to debug, and probably most of the time the threads will +be waiting for data from the server, so there is no real need to have +a truly parallel program. In this situation, coroutines are a much +better choice. They allow you to send and receive multiple requests +simultaneously, but the program flow itself is still strictly +sequential. Here's how to do it (suppose the URLs you'd like to +retrieve a stored in *url_list*):: + + import asyncio + import atexit + from urllib.parse import urlsplit, urlunsplit + + def get_url(host, port, path): + conn = HTTPConnection(host, port=port) + yield from conn.co_send_request('GET', path) + resp = yield from conn.co_read_response() + assert resp.status == 200 + body = yield from conn.co_readall() + return body + + futures = [] + for url in url_list: + o = urlsplit(url) + # Path is obtained by removing scheme, hostname and fragment + # identifier from the url + path = urlunsplit(('', '') + o[2:4] + ('',)) + + # Create a coroutine and future for each URL + futures.append(AioFuture(get_url(o.hostname, o.port, path))) + + # Run coroutines + loop = asyncio.get_event_loop() + atexit.register(loop.close) + loop.run_until_complete(asyncio.wait(futures)) + + # Get the results + bodies = [ x.result() for x in futures ] + + +When to invoke `AioFuture` +-------------------------- + +When creating your own coroutines, you generally have two choices: + +#. You can create asyncio style coroutines, in which you wrap calls to + Dugong coroutines into `AioFuture`, e.g.:: + + # ... + + @asyncio.coroutine + def do_stuff(): + # ... + yield from AioFuture(conn.co_read_response()) + # .. + buf = yield from AioFuture(conn.co_read(8192)) + # ... + + # May also call other asyncio compatible coroutines: + yield from asyncio.sleep(1) + + # .. + + task = asyncio.Task(do_stuff) + loop.run_until_complete(task) + + The advantage of this style is that even though you need to wrap + every Dugong call into `AioFuture`, you can freely mix Dugong and + other asyncio compatible coroutines. + +#. You create Dugong style coroutines, and wrap them into `AioFuture` + just before adding them to the asyncio event loop, e.g.:: + + # ... + + def do_stuff(): + # ... + yield from conn.co_read_response() + # .. + buf = yield from conn.co_read(8192) + # ... + # Other coroutines must yield PollNeeded instance, so + # we cannot yield from asyncio compatible coroutines: + #yield from asyncio.sleep(1) # WON'T WORK! + + fut = AioFuture(do_stuf()) + loop.run_until_complete(fut) + + The advantage of this is that you need to call `AioFuture` only + once. The disadvantage is that you can not yield from other asyncio + coroutines in your coroutine. + +Generally it's recommended to use the style that produces more +readable code. + + +Building your own Event-Loop +============================ + +As explained before, the easiest way to schedule coroutines is to use +the asyncio module. However, Dugong coroutines have a well-defined +interface, and you can just as well write your own coroutine +scheduling loop. In this case, the asyncio module is not used at all. + +Below is a simple example that uses this technique to switch execution +between two coroutines that send requests and read responses. The code +tries to retrieve a number of documents (stored in *path_list*), +stores the missing paths in *missing_documents*, and saves the +contents of the existing documents to disk. :: + + # Note: in a real application, don't forget to ensure that + # conn.disconnect() is called eventually + conn = HTTPConnection('somehost.com') + missing_documents = [] + + # This function returns a coroutine that sends all requests + def send_requests(): + for path in path_list: + yield from conn.co_send_request('GET', path) + + # This functions returns a coroutine that reads all responses + def read_responses(): + for (i, path) in enumerate(path_list): + resp = yield from conn.co_read_response() + if resp.status != 200: + missing_documents.append(resp.path) + with open('doc_%i.dat' % i, 'wb') as fh: + buf = yield from conn.readall() + fh.write(buf) + + # Create coroutines + send_request_crt = send_requests() + read_response_crt = read_responses() + + while True: + # Send requests until we block + if send_request_crt: + try: + io_req_1 = next(send_request_crt) + except StopIteration: + # All requests sent + send_request_crt = None + + # Read responses until we block + try: + io_req_2 = next(read_response_crt) + except StopIteration as exc: + # All responses read + break + + # Wait for fds to become ready for I/O + assert io_req_1.mask == POLLOUT + assert io_req_2.mask == POLLIN + select((io_req_2.fd,), (io_req_1.fd,), ()) + + +.. _Wikipedia_Coroutine: http://en.wikipedia.org/wiki/Coroutine +.. _`PEP 342`: http://legacy.python.org/dev/peps/pep-0342/ +.. _`PEP 380`: http://legacy.python.org/dev/peps/pep-0380/ +.. _`dabeaz.com`: http://dabeaz.com/coroutines/ diff --git a/rst/index.rst b/rst/index.rst new file mode 100644 index 0000000..d91adaf --- /dev/null +++ b/rst/index.rst @@ -0,0 +1,23 @@ +========================== + The Python Dugong Module +========================== + +Table of Contents +================= + +.. toctree:: + :maxdepth: 2 + + intro.rst + tutorial.rst + api.rst + coroutines.rst + issues.rst + whatsnew.rst + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`search` + diff --git a/rst/intro.rst b/rst/intro.rst new file mode 100644 index 0000000..4fc4f18 --- /dev/null +++ b/rst/intro.rst @@ -0,0 +1,8 @@ +============== + Introduction +============== + +.. currentmodule:: dugong + +.. include:: ../README.rst + :start-after: start-intro diff --git a/rst/issues.rst b/rst/issues.rst new file mode 100644 index 0000000..8f6eeeb --- /dev/null +++ b/rst/issues.rst @@ -0,0 +1,11 @@ +Known Issues +============ + +.. currentmodule:: dugong + +* When sending requests with a ``Range`` header, Dugong may be unable + to parse the server response. This happens if the server sends + ``multipart/byteranges`` data without a ``Content-Length`` + header. This is allowed by RFC 2616, because the content length is + implicit in the multipart body, but Dugong does not yet support + parsing the body. diff --git a/rst/tutorial.rst b/rst/tutorial.rst new file mode 100644 index 0000000..065665e --- /dev/null +++ b/rst/tutorial.rst @@ -0,0 +1,296 @@ +========== + Tutorial +========== + +.. currentmodule:: dugong + + +Basic Use +========= + +A HTTP request can be send and read in four lines:: + + with HTTPConnection('www.python.org') as conn: + conn.send_request('GET', '/index.html') + resp = conn.read_response() + body = conn.readall() + +`~HTTPConnection.send_request` is a `HTTPResponse` object that gives +access to the response header:: + + print('Server said:') + print('%03d %s' % (resp.status, resp.reason)) + for (key, value) in resp.headers.items(): + print('%s: %s' % (key, value)) + +`HTTPConnection.readall` returns a a :term:`bytes-like object`. To +convert to text, you could do something like :: + + hit = re.match(r'^(.+?)(?:; charset=(.+))?$', resp.headers['Content-Type']) + if not hit: + raise SystemExit("Can't determine response charset") + elif hit.group(2): # found explicity charset + charset = hit.group(2) + elif hit.group(1).startswith('text/'): + charset = 'latin1' # default for text/ types by RFC 2616 + else: + raise SystemExit('Server sent binary data') + text_body = body.decode(charset) + + +SSL Connections +=============== + +If you would like to establish a secure connection, you need to pass +the appropriate `~ssl.SSLContext` object to `HTTPConnection`. For example:: + + ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ssl_context.options |= ssl.OP_NO_SSLv2 + ssl_context.verify_mode = ssl.CERT_REQUIRED + ssl_context.set_default_verify_paths() + + with HTTPConnection('www.google.com', ssl_context=ssl_context) as conn: + conn.send_request('GET', '/index.html') + resp = conn.read_response() + body = conn.readall() + +If you need information about the peer certificate, use the +`~HTTPConnection.get_ssl_peercert` method. + +Streaming API +============= + +When retrieving larger objects, it's generally better not to read the +response body all at once but in smaller chunks:: + + BUFSIZE = 32*1024 # Read in 32 kB chunks + + # ... + + conn.send_request('GET', '/big_movie.mp4') + resp = conn.read_response() + assert resp.status == 200 + + with open('big_movie.mp4', 'wb') as fh: + while True: + buf = conn.read(BUFSIZE) + if not buf: + break + fh.write(buf) + +Alternatively, the `~HTTPConnection.readinto` method may give better +performance in some situations:: + + buf = bytearray(BUFSIZE) + with open('big_movie.mp4', 'wb') as fh: + while True: + len_ = conn.readinto(buf) + if not len_: + break + fh.write(buf[:len_]) + + +Uploading Data +============== + +If you want to send data to the server, you can provide the data +directly to `~HTTPConnection.send_request`, :: + + # A simple SQL injection attack for your favorite PHP script + request_body = "'; DELETE FROM passwords;".encode('us-ascii') + with HTTPConnection('somehost.com') as conn: + conn.send_request('POST', '/form.php', body=request_body) + conn.read_response() + +or (if you want to send bigger amounts) you can provide it in multiple +chunks:: + + # ... + with open('newest_release.mp4', r'b') as fh: + size = os.fstat(fh.fileno()).st_size + conn.send_request('PUT', '/public/newest_release.mp4', + body=BodyFollowing(size)) + + while True: + buf = fh.read(BUFSIZE) + if not buf: + break + conn.write(buf) + + resp = conn.read_response() + assert resp.status in (200, 204) + +Here we used the special `BodyFollowing` class to indicate that the +request body data will be provided in separate calls. + +100-Continue Support +==================== + +When having to transfer large amounts of request bodies to the server, +you typically do not want to sent all the data over the network just +to find out that the server rejected the request because of +e.g. insufficient permissions. To avoid this situation, HTTP 1.1 +specifies the *100-continue* mechanism. When using 100-continue, the +client transmits an additional ``Expect: 100-continue`` request +header, and then waits for the server to reply with status ``100 +Continue`` before sending the request body data. If the server instead +responds with an error, the client can avoid pointless transmission of +the request body. + +To use this mechanism, pass the *expect100* parameter to +`~HTTPConnection.send_request`, and call +`~HTTPConnection.read_response` twice: once before sending body data, +and a second time to read the final response:: + + # ... + with open('newest_release.mp4', r'b') as fh: + size = os.fstat(fh.fileno()).st_size + conn.send_request('PUT', '/public/newest_release.mp4', + body=BodyFollowing(size), expect100=True) + + resp = conn.read_response() + if resp.status != 100: + raise RuntimeError('Server said: %s' % resp.reason) + + while True: + buf = fh.read(BUFSIZE) + if not buf: + break + conn.write(buf) + + resp = conn.read_response() + assert resp.status in (200, 204) + + +Retrying on Error +================= + +Sometimes the connection to the remote server may get interrupted for +a variety of reasons, resulting in a variety of exceptions. For +convience, you may use the `is_temp_network_error` method to determine if a +given exception indicates a temporary problem (i.e., if it makes sense +to retry):: + + delay = 1 + conn = HTTPConnection('www.python.org') + while True: + try: + conn.send_request('GET', '/index.html') + conn.read_response() + body = conn.readall() + except Exception as exc: + if is_temp_network_error(exc): + print('Got %s, retrying..' % exc) + time.sleep(delay) + delay *= 2 + else: + raise + else: + break + finally: + conn.disconnect() + + +Timing out +========== + +It can take quite a long time before the operation system recognises +that a TCP/IP connection has been interrupted. If you'd rather be +informed right away when there has been no data exchange for some +period of time, dugong allows you to specify a custom timeout:: + + conn = HTTPConnection('www.python.org') + conn.timeout = 10 + try: + conn.send_request('GET', '/index.html') + conn.read_response() + body = conn.readall() + except ConnectionTimedOut: + print('Unable to send or receive any data for more than', + conn.timeout, 'seconds, aborting.') + sys.exit(1) + + +.. _pipelining: + +Pipelining with Threads +======================= + +Pipelining means sending multiple requests in succession, without +waiting for the responses. First, let's consider how do **not** do it:: + + # DO NOT DO THIS! + conn = HTTPConnection('somehost.com') + for path in path_list: + conn.send_request('GET', path) + + bodies = [] + for path in path_list: + resp = conn.read_response() + assert resp.status == 200 + bodies.append(conn.readall()) + +This will probably even work as long as you don't have too many +elements in *path_list*. However, it is very bad practice, because at +some point the server will stop reading requests until some responses +have been read (because all the TCP buffers are full). At this point, +your application will deadlock. + +One better way do it is to use threads. Dugong is not generally +threadsafe, but using one thread to send requests and one thread to +read responses is supported:: + + with HTTPConnection('somehost.com') as conn: + + def send_requests(): + for path in path_list: + conn.send_request('GET', path) + thread = threading.thread(target=send_requests) + thread.run() + + bodies = [] + for path in path_list: + resp = conn.read_response() + assert resp.status == 200 + bodies.append(conn.readall()) + + thread.join() + +Another way is to use coroutines. This is explained in the next +section. + + +.. _coroutine_pipelining: + +Pipelining with Coroutines +========================== + +Instead of using two threads to send requests and responses, you can +also use two coroutines. A coroutine is essentially a function that +can be suspended and resumed at specific points. Dugong coroutines +suspend themself when they would have to wait for an I/O operation to +complete. This makes them perfect for pipelining: we'll define one +coroutine that sends requests, and a second one to read responses, and +then execute them "interleaved": whenever we can't send another +request, we try to read a response, and if we can't read a response, +we try to send another request. + +The following example demonstrates how to do this to efficiently +retrieve a large number of documents (stored in *url_list*): + +.. literalinclude:: ../examples/pipeline1.py + :start-after: start-example + :end-before: end-example + +Here we have used the :ref:`yield from expression ` to +integrate the coroutines returned by +`~HTTPConnection.co_send_request`, `~HTTPConnection.co_read_response`, +and `~HTTPConnection.co_readall` into two custom coroutines +*send_requests* and *read_responses*. To schedule the coroutines, we +use `AioFuture` to obtain `asyncio Futures ` for them, +and then rely on the :mod:`asyncio` module to do the heavy lifting and +switch execution between them at the right times. + +For more details about this, take a look at :ref:`coroutines`, or the +`asyncio documentation `. + diff --git a/rst/whatsnew.rst b/rst/whatsnew.rst new file mode 100644 index 0000000..0dca305 --- /dev/null +++ b/rst/whatsnew.rst @@ -0,0 +1,6 @@ +=========== + Changelog +=========== + +.. include:: ../Changes.rst + diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..eea4f30 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,12 @@ +[sdist] +formats = bztar + +[build_sphinx] +source-dir = rst +build-dir = doc + +[egg_info] +tag_build = +tag_date = 0 +tag_svn_revision = 0 + diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..78afb4a --- /dev/null +++ b/setup.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 + +import sys +import os.path +import warnings + +try: + import setuptools +except ImportError: + raise SystemExit('Setuptools/distribute package not found. Please install from ' + 'https://pypi.python.org/pypi/distribute') + +if sys.version_info < (3,3): + raise SystemExit('Python version is %d.%d.%d, but Dugong requires 3.3 or newer' + % sys.version_info[:3]) + +basedir = os.path.abspath(os.path.dirname(sys.argv[0])) +if os.path.exists(os.path.join(basedir, 'MANIFEST.in')): + print('found MANIFEST.in, running in developer mode') + warnings.resetwarnings() + # We can't use `error`, because e.g. Sphinx triggers a + # DeprecationWarning. + warnings.simplefilter('default') + +def main(): + try: + from sphinx.application import Sphinx #pylint: disable-msg=W0612 + except ImportError: + pass + else: + fix_docutils() + + with open(os.path.join(basedir, 'README.rst'), 'r') as fh: + long_desc = fh.read() + import dugong + + setuptools.setup( + name='dugong', + zip_safe=True, + long_description=long_desc, + version=dugong.__version__, + description=('A HTTP 1.1 client module supporting asynchronous IO, pipelining ' + 'and `Expect: 100-continue`. Designed for RESTful protocols.'), + author='Nikolaus Rath', + author_email='Nikolaus@rath.org', + license='PSF', + keywords=['http'], + package_dir={'': '.'}, + packages=setuptools.find_packages(), + url='https://bitbucket.org/nikratio/python-dugong', + classifiers=['Programming Language :: Python :: 3', + 'Development Status :: 5 - Production/Stable', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Python Software Foundation License', + 'Topic :: Internet :: WWW/HTTP', + 'Topic :: Software Development :: Libraries :: Python Modules' ], + provides=['dugong'], + command_options={ 'sdist': { 'formats': ('setup.py', 'bztar') } , + 'build_sphinx': {'version': ('setup.py', dugong.__version__), + 'release': ('setup.py', dugong.__version__) }}, + ) + + +def fix_docutils(): + '''Work around https://bitbucket.org/birkenfeld/sphinx/issue/1154/''' + + import docutils.parsers + from docutils.parsers import rst + old_getclass = docutils.parsers.get_parser_class + + # Check if bug is there + try: + old_getclass('rst') + except AttributeError: + pass + else: + return + + def get_parser_class(parser_name): + """Return the Parser class from the `parser_name` module.""" + if parser_name in ('rst', 'restructuredtext'): + return rst.Parser + else: + return old_getclass(parser_name) + docutils.parsers.get_parser_class = get_parser_class + + assert docutils.parsers.get_parser_class('rst') is rst.Parser + +if __name__ == '__main__': + main() diff --git a/test/ca.crt b/test/ca.crt new file mode 100644 index 0000000..fb7d23a --- /dev/null +++ b/test/ca.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSzCCAgOgAwIBAgIEUwLoODANBgkqhkiG9w0BAQsFADATMREwDwYDVQQDEwhk +dW1teSBjYTAiGA8yMDE0MDIxODA0NTcyOVoYDzIwNDEwNzA1MDQ1NzMxWjATMREw +DwYDVQQDEwhkdW1teSBjYTCCAVIwDQYJKoZIhvcNAQEBBQADggE/ADCCAToCggEx +AJgUz8OOka9IuNPxF9ZIO8V+RlXH71g/ahKBMJcDufZAzjMOyvb7ak0VfRQpicS1 +i67vWViCJo8gJ+FENF5vlDBzQiSZJ6el9+Ea1mulsO93sHDSYfibfZ/WVR9pL8e6 +HwkLTBGRcxcxfILuZn4I8l4ykHw3TKrqHg8O9ZBoBJZ8lt60AoSGUoRMFtKhuVQL +U+8HTF/cZQ9ibwBbDRhyFVCWA0MK9TYFJRPAoyT/3w3PoekWr1ONA5+0qS2fQ4H7 +DXy+ZH90eQ2uEXNOQUS0uE/1y5xfUK2bRz/NOHN55iSG32WaNzEzX/uVvH8IcBdK +mq3QijOOluhdjkGlmLTFyQUTDzPTYRePWzqyZstgGsYAb0gV+hj3LR2UugVQN56F +cLjNGG0KMw2ci9ObI7OU0/MCAwEAAaNDMEEwDwYDVR0TAQH/BAUwAwEB/zAPBgNV +HQ8BAf8EBQMDBwQAMB0GA1UdDgQWBBReNxD2+f7CN2x1xRj4k2seW4eyWzANBgkq +hkiG9w0BAQsFAAOCATEAJ1Vx5WXRpMyW57VpuREaBu70fl/MHKuA+KE28EvxhZFm +jT7PrBC6IhgofyD7dxFSPb50djBmvNcVUmldMRA4Sid4VGtl05wcCLXwEwbg4IZB +DRuKI9RT7+xeisORWee9hlPzYn0UVJoNqPmNauO1MISagN+nd4QtbRxgDMfCnbc7 +bZbNeoT/yZ1mwQJGMLKWpxd8MUb+bcVxhq4/NctoKdYmmWI1y2GeTrEwhSDzi23+ ++MsGbKK/7NjNOtIeHSqMA8PiLA0MP9JT5Ul7qyk2ZRHCcRDFgOQK5YYjhCPxYwJQ +d+5bvkKHoRpjzMGj3xfXq1pqZSya8PW2Zf0r7805DE8C5a7mMlgmtW3YPEhPjQ7v +pgIOCSZSjG674vDQKCs1dhPNVZNzdM9MMJwX1jofzA== +-----END CERTIFICATE----- diff --git a/test/ca.key b/test/ca.key new file mode 100644 index 0000000..9cc300b --- /dev/null +++ b/test/ca.key @@ -0,0 +1,165 @@ +Public Key Info: + Public Key Algorithm: RSA + Key Security Level: Normal (2432 bits) + +modulus: + 00:98:14:cf:c3:8e:91:af:48:b8:d3:f1:17:d6:48: + 3b:c5:7e:46:55:c7:ef:58:3f:6a:12:81:30:97:03: + b9:f6:40:ce:33:0e:ca:f6:fb:6a:4d:15:7d:14:29: + 89:c4:b5:8b:ae:ef:59:58:82:26:8f:20:27:e1:44: + 34:5e:6f:94:30:73:42:24:99:27:a7:a5:f7:e1:1a: + d6:6b:a5:b0:ef:77:b0:70:d2:61:f8:9b:7d:9f:d6: + 55:1f:69:2f:c7:ba:1f:09:0b:4c:11:91:73:17:31: + 7c:82:ee:66:7e:08:f2:5e:32:90:7c:37:4c:aa:ea: + 1e:0f:0e:f5:90:68:04:96:7c:96:de:b4:02:84:86: + 52:84:4c:16:d2:a1:b9:54:0b:53:ef:07:4c:5f:dc: + 65:0f:62:6f:00:5b:0d:18:72:15:50:96:03:43:0a: + f5:36:05:25:13:c0:a3:24:ff:df:0d:cf:a1:e9:16: + af:53:8d:03:9f:b4:a9:2d:9f:43:81:fb:0d:7c:be: + 64:7f:74:79:0d:ae:11:73:4e:41:44:b4:b8:4f:f5: + cb:9c:5f:50:ad:9b:47:3f:cd:38:73:79:e6:24:86: + df:65:9a:37:31:33:5f:fb:95:bc:7f:08:70:17:4a: + 9a:ad:d0:8a:33:8e:96:e8:5d:8e:41:a5:98:b4:c5: + c9:05:13:0f:33:d3:61:17:8f:5b:3a:b2:66:cb:60: + 1a:c6:00:6f:48:15:fa:18:f7:2d:1d:94:ba:05:50: + 37:9e:85:70:b8:cd:18:6d:0a:33:0d:9c:8b:d3:9b: + 23:b3:94:d3:f3: + +public exponent: + 01:00:01: + +private exponent: + 30:48:93:39:ec:1e:b6:be:e8:e7:69:28:ff:40:49: + b1:c7:08:7f:9d:8d:90:bf:f9:66:4c:9b:e7:a6:28: + 39:55:93:e1:c3:f5:8e:7a:7d:e2:61:4b:27:c6:94: + 55:de:a9:ad:6d:92:39:36:81:15:79:c9:0f:8c:fe: + ef:68:8f:6c:de:7b:06:71:95:94:75:6e:00:8a:eb: + be:9b:89:54:4d:f4:b7:ad:23:ab:b1:7a:2d:ef:f7: + 70:94:8d:b5:e4:92:8b:e5:89:4e:1e:96:a2:ab:74: + 73:81:4b:0b:02:be:28:47:f7:75:68:7d:2f:da:4a: + 96:cf:52:09:79:ec:d2:f6:55:53:f9:64:bf:3c:82: + 20:3c:a4:16:55:80:28:7d:bc:35:2c:65:04:1e:4d: + fb:96:f0:88:01:01:f1:a4:b4:a2:e3:df:31:90:2a: + c1:0f:a7:81:09:f8:5a:21:c4:75:52:58:f4:d5:29: + 6d:cd:55:76:f2:60:de:e7:5c:9a:98:2d:11:94:54: + c2:7d:4a:01:0c:29:f2:7a:a8:de:e1:80:53:bd:b1: + d1:bc:87:d3:67:8d:a9:5c:1e:6a:85:2c:2a:f7:bd: + ca:7e:50:33:30:be:d3:0b:fb:3d:77:df:6d:4d:1b: + d7:4d:4b:8c:d6:46:7c:f0:40:88:e3:57:80:31:38: + a7:f5:f9:da:7f:d3:72:d6:a8:4c:58:cb:97:44:37: + 9a:17:30:6e:21:af:ff:38:50:9b:45:93:41:a3:5f: + e6:82:1d:b5:23:62:26:aa:bd:dd:af:e7:6a:67:0a: + 12:91:37:e9: + +prime1: + 00:c2:cc:69:2f:c1:2f:e3:cc:6c:a8:36:df:5c:09: + 93:95:b8:70:c3:20:82:e1:d0:ea:b8:09:7b:08:28: + e2:b2:26:dd:70:9c:b2:bc:68:43:ae:06:d7:4a:b5: + 09:36:72:cb:8d:66:d9:e8:b7:81:d4:ad:73:04:91: + b7:d2:a9:01:bc:bc:f3:a6:17:b1:71:a2:8a:bf:35: + a8:55:0e:a1:41:6e:6d:e3:db:30:76:7e:69:43:9b: + ec:54:99:e6:94:ea:a1:06:2e:b6:07:74:fc:80:f0: + f7:2f:ae:ef:dd:f2:14:50:da:76:7d:3d:b3:0f:42: + 84:97:49:21:ce:f9:ec:20:32:40:88:92:4d:0c:1d: + b1:55:30:ee:f5:0d:c1:d8:14:e0:f3:97:36:75:ac: + 4c:b0:d7: + +prime2: + 00:c7:dc:ab:47:9f:8b:d6:cb:7a:9e:9d:00:d6:ef: + 43:eb:4e:0e:25:0a:09:21:b7:80:60:b9:3b:56:6b: + 3c:bd:52:4e:d3:76:df:dc:73:63:25:9c:bd:79:41: + 94:0a:56:34:4f:55:13:2b:0b:df:1f:c2:60:74:bc: + e4:3a:5c:8d:9e:3c:1b:6f:75:31:e6:7f:99:5e:3b: + 30:63:9f:ac:88:ee:32:36:f0:f2:d8:e4:67:b7:bf: + 03:56:d4:9a:d0:7a:a2:0c:68:c4:53:f2:40:b0:7b: + f7:ba:42:12:73:73:82:29:a0:7b:66:3f:c8:d7:1f: + 0d:6c:dd:6c:d7:10:85:69:52:f2:78:cb:81:0a:7b: + 60:bc:07:06:18:b7:79:cf:1f:c9:7f:15:6c:1e:1b: + 38:e6:45: + +coefficient: + 00:8f:5c:a0:11:95:68:bb:7e:a0:6b:f2:c6:8a:3b: + 86:3a:64:cf:1f:42:7e:a9:44:4a:cb:c9:f1:a4:82: + 92:da:ae:99:2d:31:b6:8e:ea:ac:fc:a7:5f:4d:90: + a4:64:20:e4:b5:99:35:a7:18:c3:55:dd:64:1f:ad: + 13:eb:a8:9b:24:f4:4d:c7:71:35:59:cf:41:15:04: + 10:f1:3c:5e:30:91:63:fe:76:16:09:35:e4:2f:12: + 24:c2:20:71:de:6d:34:e4:a5:90:b4:ac:c5:80:a4: + 67:9e:e7:74:91:92:44:ac:e5:0e:cf:01:5e:ce:d2: + 44:60:c0:3a:98:60:ef:10:c5:72:fa:d3:db:5d:93: + 78:42:0a:58:50:fa:9d:70:c4:86:87:d7:f5:63:98: + 62:3b:1c: + +exp1: + 51:fd:37:48:a2:47:45:da:04:e1:c8:36:ba:c6:4d: + 17:f7:49:7e:d7:70:3f:1b:6f:af:86:4e:02:61:33: + 09:48:d2:6b:53:88:e7:43:fb:38:84:28:99:89:19: + 17:91:b9:9b:0a:6f:2d:44:0f:a1:34:5f:f4:cc:60: + 52:8a:4f:f0:e0:96:ac:91:cc:5d:c7:cc:1e:2e:b7: + 6b:15:7d:49:cc:f0:f3:b6:8d:ef:51:c5:7c:6f:64: + 49:37:7d:95:b0:2b:96:2e:92:ef:10:8e:36:b7:35: + 53:1c:8e:59:1c:4a:f0:bd:02:a2:34:15:e9:96:55: + b8:57:4f:a3:8e:0d:94:7f:92:29:e9:6a:04:6f:7a: + f4:20:64:73:40:17:16:9b:b3:12:d3:d2:58:34:ed: + 12:81: + +exp2: + 60:f7:1a:d2:61:01:c3:70:6d:49:4e:fa:fd:4c:90: + 33:35:67:7f:68:e3:0d:4c:ae:28:3f:36:1e:b3:60: + 80:a9:d2:3c:9e:4b:f5:f4:b5:81:a6:0d:f7:2a:6d: + dc:a5:fe:33:0f:1f:81:9e:fc:dd:b7:bc:7c:66:b8: + 83:e8:2b:7d:3d:c4:41:cb:26:2e:a2:71:92:5a:3d: + 1b:d8:78:28:e7:07:cd:c8:10:ca:51:e6:50:2b:88: + 3f:34:5e:f8:0e:c7:58:25:ec:3a:9e:29:ec:75:f8: + b6:91:1f:ca:8b:9b:f6:fe:39:60:5e:49:de:b4:de: + 1a:97:43:1b:04:94:f5:88:9c:c3:26:58:b0:f1:32: + f9:86:9e:da:97:09:ae:07:ca:06:b2:2a:06:61:46: + b5:d1: + + +Public Key ID: 5E:37:10:F6:F9:FE:C2:37:6C:75:C5:18:F8:93:6B:1E:5B:87:B2:5B +Public key's random art: ++--[ RSA 2432]----+ +| o . | +| . o... | +| . o. = | +| . .= o| +| S . o .+.| +| . . ..o= =| +| . *E++| +| ..==.| +| ...oo| ++-----------------+ + +-----BEGIN RSA PRIVATE KEY----- +MIIFewIBAAKCATEAmBTPw46Rr0i40/EX1kg7xX5GVcfvWD9qEoEwlwO59kDOMw7K +9vtqTRV9FCmJxLWLru9ZWIImjyAn4UQ0Xm+UMHNCJJknp6X34RrWa6Ww73ewcNJh ++Jt9n9ZVH2kvx7ofCQtMEZFzFzF8gu5mfgjyXjKQfDdMquoeDw71kGgElnyW3rQC +hIZShEwW0qG5VAtT7wdMX9xlD2JvAFsNGHIVUJYDQwr1NgUlE8CjJP/fDc+h6Rav +U40Dn7SpLZ9DgfsNfL5kf3R5Da4Rc05BRLS4T/XLnF9QrZtHP804c3nmJIbfZZo3 +MTNf+5W8fwhwF0qardCKM46W6F2OQaWYtMXJBRMPM9NhF49bOrJmy2AaxgBvSBX6 +GPctHZS6BVA3noVwuM0YbQozDZyL05sjs5TT8wIDAQABAoIBMDBIkznsHra+6Odp +KP9ASbHHCH+djZC/+WZMm+emKDlVk+HD9Y56feJhSyfGlFXeqa1tkjk2gRV5yQ+M +/u9oj2zeewZxlZR1bgCK676biVRN9LetI6uxei3v93CUjbXkkovliU4elqKrdHOB +SwsCvihH93VofS/aSpbPUgl57NL2VVP5ZL88giA8pBZVgCh9vDUsZQQeTfuW8IgB +AfGktKLj3zGQKsEPp4EJ+FohxHVSWPTVKW3NVXbyYN7nXJqYLRGUVMJ9SgEMKfJ6 +qN7hgFO9sdG8h9NnjalcHmqFLCr3vcp+UDMwvtML+z13321NG9dNS4zWRnzwQIjj +V4AxOKf1+dp/03LWqExYy5dEN5oXMG4hr/84UJtFk0GjX+aCHbUjYiaqvd2v52pn +ChKRN+kCgZkAwsxpL8Ev48xsqDbfXAmTlbhwwyCC4dDquAl7CCjisibdcJyyvGhD +rgbXSrUJNnLLjWbZ6LeB1K1zBJG30qkBvLzzphexcaKKvzWoVQ6hQW5t49swdn5p +Q5vsVJnmlOqhBi62B3T8gPD3L67v3fIUUNp2fT2zD0KEl0khzvnsIDJAiJJNDB2x +VTDu9Q3B2BTg85c2daxMsNcCgZkAx9yrR5+L1st6np0A1u9D604OJQoJIbeAYLk7 +Vms8vVJO03bf3HNjJZy9eUGUClY0T1UTKwvfH8JgdLzkOlyNnjwbb3Ux5n+ZXjsw +Y5+siO4yNvDy2ORnt78DVtSa0HqiDGjEU/JAsHv3ukISc3OCKaB7Zj/I1x8NbN1s +1xCFaVLyeMuBCntgvAcGGLd5zx/JfxVsHhs45kUCgZhR/TdIokdF2gThyDa6xk0X +90l+13A/G2+vhk4CYTMJSNJrU4jnQ/s4hCiZiRkXkbmbCm8tRA+hNF/0zGBSik/w +4Jaskcxdx8weLrdrFX1JzPDzto3vUcV8b2RJN32VsCuWLpLvEI42tzVTHI5ZHErw +vQKiNBXpllW4V0+jjg2Uf5Ip6WoEb3r0IGRzQBcWm7MS09JYNO0SgQKBmGD3GtJh +AcNwbUlO+v1MkDM1Z39o4w1Mrig/Nh6zYICp0jyeS/X0tYGmDfcqbdyl/jMPH4Ge +/N23vHxmuIPoK309xEHLJi6icZJaPRvYeCjnB83IEMpR5lAriD80XvgOx1gl7Dqe +Kex1+LaRH8qLm/b+OWBeSd603hqXQxsElPWInMMmWLDxMvmGntqXCa4HygayKgZh +RrXRAoGZAI9coBGVaLt+oGvyxoo7hjpkzx9CfqlESsvJ8aSCktqumS0xto7qrPyn +X02QpGQg5LWZNacYw1XdZB+tE+uomyT0TcdxNVnPQRUEEPE8XjCRY/52Fgk15C8S +JMIgcd5tNOSlkLSsxYCkZ57ndJGSRKzlDs8BXs7SRGDAOphg7xDFcvrT212TeEIK +WFD6nXDEhofX9WOYYjsc +-----END RSA PRIVATE KEY----- diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..a09497f --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,65 @@ +import sys +import os.path +import logging +import gc + +if sys.version_info < (3,3): + raise SystemExit('Python version is %d.%d.%d, but Dugong requires 3.3 or newer' + % sys.version_info[:3]) +# Enable output checks +pytest_plugins = ('pytest_checklogs') + +def pytest_addoption(parser): + group = parser.getgroup("general") + group._addoption("--installed", action="store_true", default=False, + help="Test the installed package.") + + group = parser.getgroup("terminal reporting") + group._addoption("--logdebug", action="append", metavar='', + help="Activate debugging output from for tests. Use `all` " + "to get debug messages from all modules. This option can be " + "specified multiple times.") + +def pytest_configure(config): + # If we are running from the source directory, make sure that we load + # modules from here + basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..')) + if not config.getoption('installed'): + if (os.path.exists(os.path.join(basedir, 'setup.py')) and + os.path.exists(os.path.join(basedir, 'dugong', '__init__.py'))): + sys.path.insert(0, basedir) + + # Make sure that called processes use the same path + pp = os.environ.get('PYTHONPATH', None) + if pp: + pp = '%s:%s' % (basedir, pp) + else: + pp = basedir + os.environ['PYTHONPATH'] = pp + + # When running from VCS repo, enable all warnings + if os.path.exists(os.path.join(basedir, 'MANIFEST.in')): + import warnings + warnings.resetwarnings() + warnings.simplefilter('default') + + # Configure logging. We don't set a default handler but rely on + # the catchlog pytest plugin. + logdebug = config.getoption('logdebug') + root_logger = logging.getLogger() + if logdebug is not None: + logging.disable(logging.NOTSET) + if 'all' in logdebug: + root_logger.setLevel(logging.DEBUG) + else: + for module in logdebug: + logging.getLogger(module).setLevel(logging.DEBUG) + else: + root_logger.setLevel(logging.INFO) + logging.disable(logging.DEBUG) + logging.captureWarnings(capture=True) + +# Run gc.collect() at the end of every test, so that we get ResourceWarnings +# as early as possible. +def pytest_runtest_teardown(item, nextitem): + gc.collect() diff --git a/test/pytest.ini b/test/pytest.ini new file mode 100644 index 0000000..0431cd1 --- /dev/null +++ b/test/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +addopts = --verbose --assert=rewrite --tb=native diff --git a/test/pytest_checklogs.py b/test/pytest_checklogs.py new file mode 100644 index 0000000..046df20 --- /dev/null +++ b/test/pytest_checklogs.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +''' +py.test plugin to look for suspicious phrases in messages +emitted on stdout/stderr or via the logging module. + +False positives can be registered via a new `reg_output` +fixture (for messages to stdout/stderr), and a `assert_logs` +function (for logging messages). +''' + +import pytest +import re +import functools +import sys +import logging +from contextlib import contextmanager + +def pytest_configure(config): + if not config.pluginmanager.hasplugin('pytest_catchlog'): + raise ImportError('pytest catchlog plugin not found') + +# Fail tests if they result in log messages of severity WARNING or more. +def check_test_log(caplog): + for record in caplog.records: + if (record.levelno >= logging.WARNING and + not getattr(record, 'checklogs_ignore', False)): + raise AssertionError('Logger received warning messages') + +class CountMessagesHandler(logging.Handler): + def __init__(self, level=logging.NOTSET): + super().__init__(level) + self.count = 0 + + def emit(self, record): + self.count += 1 + +@contextmanager +def assert_logs(pattern, level=logging.WARNING, count=None): + '''Assert that suite emits specified log message + + *pattern* is matched against the *unformatted* log message, i.e. before any + arguments are merged. + + If *count* is not None, raise an exception unless exactly *count* matching + messages are caught. + + Matched log records will also be flagged so that the caplog fixture + does not generate exceptions for them (no matter their severity). + ''' + + def filter(record): + if (record.levelno == level and + re.search(pattern, record.msg)): + record.checklogs_ignore = True + return True + return False + + handler = CountMessagesHandler() + handler.setLevel(level) + handler.addFilter(filter) + logger = logging.getLogger() + logger.addHandler(handler) + try: + yield + + finally: + logger.removeHandler(handler) + + if count is not None and handler.count != count: + raise AssertionError('Expected to catch %d %r messages, but got only %d' + % (count, pattern, handler.count)) + +def check_test_output(capfd, item): + (stdout, stderr) = capfd.readouterr() + + # Write back what we've read (so that it will still be printed) + sys.stdout.write(stdout) + sys.stderr.write(stderr) + + # Strip out false positives + try: + false_pos = item.checklogs_fp + except AttributeError: + false_pos = () + for (pattern, flags, count) in false_pos: + cp = re.compile(pattern, flags) + (stdout, cnt) = cp.subn('', stdout, count=count) + if count == 0 or count - cnt > 0: + stderr = cp.sub('', stderr, count=count - cnt) + + for pattern in ('exception', 'error', 'warning', 'fatal', 'traceback', + 'fault', 'crash(?:ed)?', 'abort(?:ed)', 'fishy'): + cp = re.compile(r'\b{}\b'.format(pattern), re.IGNORECASE | re.MULTILINE) + hit = cp.search(stderr) + if hit: + raise AssertionError('Suspicious output to stderr (matched "%s")' % hit.group(0)) + hit = cp.search(stdout) + if hit: + raise AssertionError('Suspicious output to stdout (matched "%s")' % hit.group(0)) + +def register_output(item, pattern, count=1, flags=re.MULTILINE): + '''Register *pattern* as false positive for output checking + + This prevents the test from failing because the output otherwise + appears suspicious. + ''' + + item.checklogs_fp.append((pattern, flags, count)) + +@pytest.fixture() +def reg_output(request): + assert not hasattr(request.node, 'checklogs_fp') + request.node.checklogs_fp = [] + return functools.partial(register_output, request.node) + +def check_output(item): + pm = item.config.pluginmanager + cm = pm.getplugin('capturemanager') + check_test_output(cm._capturing, item) + check_test_log(item.catch_log_handler) + +@pytest.hookimpl(trylast=True) +def pytest_runtest_setup(item): + check_output(item) +@pytest.hookimpl(trylast=True) +def pytest_runtest_call(item): + check_output(item) +@pytest.hookimpl(trylast=True) +def pytest_runtest_teardown(item, nextitem): + check_output(item) diff --git a/test/server.crt b/test/server.crt new file mode 100644 index 0000000..54cba56 --- /dev/null +++ b/test/server.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDnTCCAlWgAwIBAgIEUwLotzANBgkqhkiG9w0BAQsFADATMREwDwYDVQQDEwhk +dW1teSBjYTAiGA8yMDE0MDIxODA0NTkzNloYDzIwNDEwNzA1MDQ1OTM4WjAUMRIw +EAYDVQQDEwlsb2NhbGhvc3QwggFSMA0GCSqGSIb3DQEBAQUAA4IBPwAwggE6AoIB +MQDaoCu2oQ9s3feSDtr0l67WrQHk5mcOGobbUQzcJRTkLLyHGRbc5tXqLQ/48UCO +CYAisW6jvrDzWgKLbNBGIC/3MJR3BOVq6VFHG/Hl3UMzbhFjBvEqbRQH6HUCDg26 +omxWWyR8MNjp+2UzTd6Wg3FAqoNYWqdl628nXN3FFfs83lzhtFOjvSw3OvzVRfi2 +FVH2QlvOfVXHnE4tOnqo/2YVW5Erom9gk/II2/UvUKdexoYbeJSma8u/Lo4dvpc1 +jYcNsEfjcglOOR9bYQmUIIxtcDmvqkrbhZ46/h0hYd/LNOFieTkUllfPsePCsDrd +sgV8QTLGiAs0CB0YxgZJHI2km7OgYQ5UY8E2pHyBYYwdTlGB37+jg8w87Vs1+4tO +bsckXeogi4AJUBBZjviTYMP5AgMBAAGjgZMwgZAwDAYDVR0TAQH/BAIwADAaBgNV +HREEEzARgglsb2NhbGhvc3SHBH8AAAEwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYD +VR0PAQH/BAUDAwegADAdBgNVHQ4EFgQUW4uW1r8tEoh+OntmOoEW8u3SWFQwHwYD +VR0jBBgwFoAUXjcQ9vn+wjdsdcUY+JNrHluHslswDQYJKoZIhvcNAQELBQADggEx +AIXHMCWvaJ3VEF81m1+MsYzcWv8amsGeXDHCZhP8ek2HCzrHplSaRp5vKuY5rd98 +acdWuz/EYRqI1zCcn2O7VVQz5W9ftOxjewhqfZ3NiwGJluPDaumclwIjosCIeTyF +BwRp0xn3LpIrKkP6bGnuOVeDcDY+ghCV52nqoDtlxYuFkPsSJ/s+5YhhSFXk6AAA +hj2Z16OgyqJm6gFbb3ZkGrzW2agg9r5Vm9D2+8MVY7gcvqUhenkZGT0qdu3sNOpq +XxjzTSt96oiSTAFFcDJHnqcQv2wUgf1Jj0jKiLCGUTvajATyVTCm9Li/q42wuzDb +HaxLdiO3zIILrgtVgtnk90U/huIswhl+a4IybXvqUWPyR1b1pEd3eNj4RFnnnIWa +jJHdXN0ksyAus8wjTuVcWME= +-----END CERTIFICATE----- diff --git a/test/server.key b/test/server.key new file mode 100644 index 0000000..97ad388 --- /dev/null +++ b/test/server.key @@ -0,0 +1,165 @@ +Public Key Info: + Public Key Algorithm: RSA + Key Security Level: Normal (2432 bits) + +modulus: + 00:da:a0:2b:b6:a1:0f:6c:dd:f7:92:0e:da:f4:97: + ae:d6:ad:01:e4:e6:67:0e:1a:86:db:51:0c:dc:25: + 14:e4:2c:bc:87:19:16:dc:e6:d5:ea:2d:0f:f8:f1: + 40:8e:09:80:22:b1:6e:a3:be:b0:f3:5a:02:8b:6c: + d0:46:20:2f:f7:30:94:77:04:e5:6a:e9:51:47:1b: + f1:e5:dd:43:33:6e:11:63:06:f1:2a:6d:14:07:e8: + 75:02:0e:0d:ba:a2:6c:56:5b:24:7c:30:d8:e9:fb: + 65:33:4d:de:96:83:71:40:aa:83:58:5a:a7:65:eb: + 6f:27:5c:dd:c5:15:fb:3c:de:5c:e1:b4:53:a3:bd: + 2c:37:3a:fc:d5:45:f8:b6:15:51:f6:42:5b:ce:7d: + 55:c7:9c:4e:2d:3a:7a:a8:ff:66:15:5b:91:2b:a2: + 6f:60:93:f2:08:db:f5:2f:50:a7:5e:c6:86:1b:78: + 94:a6:6b:cb:bf:2e:8e:1d:be:97:35:8d:87:0d:b0: + 47:e3:72:09:4e:39:1f:5b:61:09:94:20:8c:6d:70: + 39:af:aa:4a:db:85:9e:3a:fe:1d:21:61:df:cb:34: + e1:62:79:39:14:96:57:cf:b1:e3:c2:b0:3a:dd:b2: + 05:7c:41:32:c6:88:0b:34:08:1d:18:c6:06:49:1c: + 8d:a4:9b:b3:a0:61:0e:54:63:c1:36:a4:7c:81:61: + 8c:1d:4e:51:81:df:bf:a3:83:cc:3c:ed:5b:35:fb: + 8b:4e:6e:c7:24:5d:ea:20:8b:80:09:50:10:59:8e: + f8:93:60:c3:f9: + +public exponent: + 01:00:01: + +private exponent: + 3a:37:e4:ec:21:3f:a6:52:05:97:53:75:63:24:f1: + 5e:21:7c:1b:a4:6f:55:06:23:5b:4b:de:0c:d1:a3: + 1a:8f:ff:34:4a:ae:17:ed:30:91:c7:a6:35:38:a9: + 64:29:8b:81:b1:96:30:ec:9a:da:72:e1:b2:97:2b: + 6f:41:2b:04:bc:5f:0c:c2:b5:05:1f:54:91:87:13: + 87:8e:c0:52:75:c1:13:89:c9:b6:ee:8d:22:fd:f8: + 0a:b0:0b:5f:e3:d8:cd:b6:3f:a5:02:ad:00:c1:fd: + 55:08:2b:7a:11:4e:9b:55:cc:dc:3e:67:cd:70:40: + 8d:4e:e1:8f:96:26:ed:32:99:b1:50:ff:e4:de:7f: + 63:c5:c5:86:55:b7:c1:65:34:0a:4b:e9:7e:b7:49: + 9c:79:b9:14:8f:53:ba:7e:21:38:f0:bd:70:17:74: + 10:56:d4:39:e9:2e:91:b4:f5:99:4b:e0:bd:27:73: + b6:83:45:a9:27:ff:d8:77:52:3a:de:15:29:95:f3: + e4:79:83:1d:1c:85:7c:ea:7a:15:22:18:af:ec:e6: + 43:ea:1b:f7:4e:af:a9:13:c7:e3:9e:b7:ab:b2:d6: + 90:4b:f9:b5:46:c4:37:7b:2b:88:1d:0f:51:06:6c: + 59:87:d4:8f:92:9b:21:c1:da:a0:31:a1:16:0f:7d: + 2c:d9:98:be:43:f7:1d:bd:f1:e2:e7:61:05:04:bf: + 88:1d:8c:33:44:67:43:9f:57:be:29:46:33:68:d5: + 25:08:90:ed:ca:7c:84:49:43:41:6c:60:c1:8c:12: + 44:70:b8:01: + +prime1: + 00:f4:f3:4a:9d:77:7e:0f:b4:68:57:bf:75:91:6e: + 5d:34:8e:72:99:53:8d:02:f0:2b:8f:d3:84:9e:24: + e1:c2:6f:df:05:8c:4c:a9:2a:ea:94:58:87:3f:17: + ce:35:69:27:ce:61:73:ad:aa:d8:57:da:4f:82:b0: + 64:d4:98:09:90:f2:c3:da:91:18:79:ec:96:52:b2: + d0:85:b8:46:a5:4c:29:e3:39:8b:be:f8:ba:c7:8d: + bd:5e:da:f7:64:91:14:25:0c:cc:b2:6b:bc:b5:fa: + 16:96:5b:5f:2f:f8:7f:0b:d6:c6:0c:70:07:cb:b3: + 11:d5:18:15:7f:5c:38:70:ba:37:61:71:4c:b1:ea: + 75:a2:40:9a:3d:d7:18:8b:94:dd:7d:3d:55:8d:64: + b6:5c:c1: + +prime2: + 00:e4:7c:e1:21:e0:18:6e:27:e3:94:f1:96:f3:12: + 80:5e:87:e2:de:40:48:24:22:8a:62:c1:a4:81:02: + 67:a6:6f:8e:c5:4a:48:c2:4b:e0:f7:a4:28:4e:fe: + 7d:a2:dc:cc:be:da:98:5a:73:39:91:8b:98:16:37: + 10:e4:06:30:51:3b:44:bf:ab:01:1e:d6:4f:31:fd: + f4:02:9c:c9:d6:54:d5:b3:db:92:91:29:b5:27:48: + a3:30:39:9d:85:11:a8:9f:d3:ef:73:8d:cd:66:7a: + 03:7a:36:90:13:ba:76:58:89:af:8e:b0:be:c8:43: + c9:c5:73:b2:82:58:ae:c3:39:f0:4b:24:0b:5f:33: + 8a:83:34:73:8c:26:a0:e7:81:c5:af:7b:17:09:a8: + 14:5d:39: + +coefficient: + 58:e1:73:01:ed:96:a2:8b:2e:66:43:7f:cc:71:a4: + cc:67:b2:07:bd:a0:19:4e:4a:89:0f:64:95:bb:e0: + 8a:b6:2d:58:c3:db:68:96:9a:27:33:23:16:86:28: + e9:c3:8e:f7:ab:df:4a:e1:f7:de:99:48:42:28:ff: + 69:bd:c3:3f:f7:82:6c:7a:ed:70:2f:ab:07:98:78: + 18:eb:77:cb:7e:2d:a2:c1:d2:c8:0b:8d:dd:72:e0: + 3b:1f:a7:62:e7:e8:71:82:d2:c2:3f:d7:97:6f:db: + e2:ed:4d:4f:c8:d6:90:bf:41:6f:a4:33:40:ab:65: + cf:09:73:06:e1:13:46:ef:db:e6:93:49:30:5e:0b: + 0d:4d:a4:51:1d:71:f8:b5:35:be:0e:24:12:3d:73: + b4:19: + +exp1: + 27:20:fa:29:5f:5e:36:da:05:d5:06:93:9c:50:b9: + f8:dc:4c:78:2b:bd:99:db:8d:c9:e2:eb:0b:6c:2e: + d8:25:90:c5:cd:1c:ad:e5:5a:25:aa:62:a8:74:80: + 0f:4e:25:fa:b4:dc:8a:c0:80:e0:bf:d1:f5:b9:81: + d1:e8:1b:97:19:00:aa:58:85:45:6c:c2:b2:a1:37: + e8:34:80:ac:85:17:27:e2:18:6d:c0:43:ed:fe:b9: + 62:7e:ae:08:55:98:97:36:8d:38:6f:37:6d:06:6f: + 37:43:8f:58:15:65:0e:1a:17:f7:02:aa:6c:22:c5: + d5:79:8c:6d:94:e8:bb:31:34:09:8f:d4:c5:93:03: + 89:90:b2:52:f0:9a:4e:29:d0:9b:e2:01:59:9d:dd: + f7:c1: + +exp2: + 1f:af:31:89:01:0a:62:3c:25:d3:01:ad:6d:07:a3: + c5:78:12:7d:bf:6c:41:96:88:9a:29:40:26:a3:ae: + bb:e5:bc:66:9f:66:77:8b:0e:27:49:1c:4d:d7:fc: + 94:19:0a:4b:6d:04:b3:86:46:f5:67:e6:2f:2e:73: + a9:2b:32:88:11:2f:f7:64:3f:43:87:74:73:fa:43: + 5c:19:61:e3:d6:df:cb:91:27:41:fa:06:a4:eb:ed: + b5:42:48:15:ab:dd:36:4c:ad:67:d8:c2:22:f3:c8: + 87:aa:09:50:b3:0a:b6:3c:61:9b:e7:e6:8c:de:d9: + 9c:07:ef:39:24:13:17:ff:70:d2:fa:ac:99:07:0c: + ae:25:17:e0:d7:7a:78:a0:06:49:cf:47:e2:8e:ec: + 44:01: + + +Public Key ID: 5B:8B:96:D6:BF:2D:12:88:7E:3A:7B:66:3A:81:16:F2:ED:D2:58:54 +Public key's random art: ++--[ RSA 2432]----+ +| | +| E | +| . | +| . . . | +| o = .S.. | +| + = .*.. | +| . * .* o. | +| o *o= .... | +| .+@ .oo. | ++-----------------+ + +-----BEGIN RSA PRIVATE KEY----- +MIIFegIBAAKCATEA2qArtqEPbN33kg7a9Jeu1q0B5OZnDhqG21EM3CUU5Cy8hxkW +3ObV6i0P+PFAjgmAIrFuo76w81oCi2zQRiAv9zCUdwTlaulRRxvx5d1DM24RYwbx +Km0UB+h1Ag4NuqJsVlskfDDY6ftlM03eloNxQKqDWFqnZetvJ1zdxRX7PN5c4bRT +o70sNzr81UX4thVR9kJbzn1Vx5xOLTp6qP9mFVuRK6JvYJPyCNv1L1CnXsaGG3iU +pmvLvy6OHb6XNY2HDbBH43IJTjkfW2EJlCCMbXA5r6pK24WeOv4dIWHfyzThYnk5 +FJZXz7HjwrA63bIFfEEyxogLNAgdGMYGSRyNpJuzoGEOVGPBNqR8gWGMHU5Rgd+/ +o4PMPO1bNfuLTm7HJF3qIIuACVAQWY74k2DD+QIDAQABAoIBMDo35OwhP6ZSBZdT +dWMk8V4hfBukb1UGI1tL3gzRoxqP/zRKrhftMJHHpjU4qWQpi4GxljDsmtpy4bKX +K29BKwS8XwzCtQUfVJGHE4eOwFJ1wROJybbujSL9+AqwC1/j2M22P6UCrQDB/VUI +K3oRTptVzNw+Z81wQI1O4Y+WJu0ymbFQ/+Tef2PFxYZVt8FlNApL6X63SZx5uRSP +U7p+ITjwvXAXdBBW1DnpLpG09ZlL4L0nc7aDRakn/9h3UjreFSmV8+R5gx0chXzq +ehUiGK/s5kPqG/dOr6kTx+Oet6uy1pBL+bVGxDd7K4gdD1EGbFmH1I+SmyHB2qAx +oRYPfSzZmL5D9x298eLnYQUEv4gdjDNEZ0OfV74pRjNo1SUIkO3KfIRJQ0FsYMGM +EkRwuAECgZkA9PNKnXd+D7RoV791kW5dNI5ymVONAvArj9OEniThwm/fBYxMqSrq +lFiHPxfONWknzmFzrarYV9pPgrBk1JgJkPLD2pEYeeyWUrLQhbhGpUwp4zmLvvi6 +x429Xtr3ZJEUJQzMsmu8tfoWlltfL/h/C9bGDHAHy7MR1RgVf1w4cLo3YXFMsep1 +okCaPdcYi5TdfT1VjWS2XMECgZkA5HzhIeAYbifjlPGW8xKAXofi3kBIJCKKYsGk +gQJnpm+OxUpIwkvg96QoTv59otzMvtqYWnM5kYuYFjcQ5AYwUTtEv6sBHtZPMf30 +ApzJ1lTVs9uSkSm1J0ijMDmdhRGon9Pvc43NZnoDejaQE7p2WImvjrC+yEPJxXOy +gliuwznwSyQLXzOKgzRzjCag54HFr3sXCagUXTkCgZgnIPopX1422gXVBpOcULn4 +3Ex4K72Z243J4usLbC7YJZDFzRyt5VolqmKodIAPTiX6tNyKwIDgv9H1uYHR6BuX +GQCqWIVFbMKyoTfoNICshRcn4hhtwEPt/rlifq4IVZiXNo04bzdtBm83Q49YFWUO +Ghf3AqpsIsXVeYxtlOi7MTQJj9TFkwOJkLJS8JpOKdCb4gFZnd33wQKBmB+vMYkB +CmI8JdMBrW0Ho8V4En2/bEGWiJopQCajrrvlvGafZneLDidJHE3X/JQZCkttBLOG +RvVn5i8uc6krMogRL/dkP0OHdHP6Q1wZYePW38uRJ0H6BqTr7bVCSBWr3TZMrWfY +wiLzyIeqCVCzCrY8YZvn5oze2ZwH7zkkExf/cNL6rJkHDK4lF+DXenigBknPR+KO +7EQBAoGYWOFzAe2WoosuZkN/zHGkzGeyB72gGU5KiQ9klbvgirYtWMPbaJaaJzMj +FoYo6cOO96vfSuH33plIQij/ab3DP/eCbHrtcC+rB5h4GOt3y34tosHSyAuN3XLg +Ox+nYufocYLSwj/Xl2/b4u1NT8jWkL9Bb6QzQKtlzwlzBuETRu/b5pNJMF4LDU2k +UR1x+LU1vg4kEj1ztBk= +-----END RSA PRIVATE KEY----- diff --git a/test/test_aio.py b/test/test_aio.py new file mode 100644 index 0000000..b33f256 --- /dev/null +++ b/test/test_aio.py @@ -0,0 +1,66 @@ +''' +test_aio.py - Unit tests for Dugong + +Copyright © 2014 Nikolaus Rath + +This module may be distributed under the terms of the Python Software Foundation +License Version 2. The complete license text may be retrieved from +http://hg.python.org/cpython/file/65f2c92ed079/LICENSE. +''' + + +if __name__ == '__main__': + import sys + import pytest + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +import socket +from select import POLLIN +import logging +from dugong import PollNeeded + +try: + import asyncio +except ImportError: + import pytest + pytestmark = pytest.mark.skipif(True, + reason='asyncio module not available') +else: + from dugong import AioFuture + +log = logging.getLogger(__name__) + +def read(sock): + for i in range(3): + log.debug('yielding') + yield PollNeeded(sock.fileno(), POLLIN) + log.debug('trying to read') + buf = sock.recv(100) + assert buf.decode() == 'text-%03d' % i + log.debug('got: %s', buf) + +def write(sock): + for i in range(3): + log.debug('sleeping') + yield from asyncio.sleep(1) + buf = ('text-%03d' % i).encode() + log.debug('writing %s', buf) + sock.send(buf) + +def test_aio_future(): + loop = asyncio.get_event_loop() + try: + (sock1, sock2) = socket.socketpair() + + asyncio.Task(write(sock2)) + read_fut = AioFuture(read(sock1)) + read_fut.add_done_callback(lambda fut: loop.stop()) + loop.call_later(6, loop.stop) + loop.run_forever() + + assert read_fut.done() + + sock1.close() + sock2.close() + finally: + loop.close() diff --git a/test/test_dugong.py b/test/test_dugong.py new file mode 100755 index 0000000..5ef370e --- /dev/null +++ b/test/test_dugong.py @@ -0,0 +1,1174 @@ +#!/usr/bin/env python3 +''' +test_dugong.py - Unit tests for dugong.py - run with py.test + +Copyright © 2014 Nikolaus Rath + +This module may be distributed under the terms of the Python Software Foundation +License Version 2. The complete license text may be retrieved from +http://hg.python.org/cpython/file/65f2c92ed079/LICENSE. +''' + +if __name__ == '__main__': + import pytest + import sys + + # For profiling: + #import cProfile + #cProfile.run('pytest.main([%r] + sys.argv[1:])' % __file__, + # 'cProfile.dat') + #sys.exit() + + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +from dugong import (HTTPConnection, BodyFollowing, CaseInsensitiveDict, _join, + ConnectionClosed) +import dugong +from http.server import BaseHTTPRequestHandler, _quote_html +from io import TextIOWrapper +from base64 import b64encode +import http.client +import itertools +import pytest +import time +import ssl +import re +import os +import hashlib +import threading +import socketserver +from pytest import raises as assert_raises + +# We want to test with a real certificate +SSL_TEST_HOST = 'www.google.com' + +TEST_DIR = os.path.dirname(__file__) + +class HTTPServer(socketserver.TCPServer): + # Extended to add SSL support + def get_request(self): + (sock, addr) = super().get_request() + if self.ssl_context: + sock = self.ssl_context.wrap_socket(sock, server_side=True) + return (sock, addr) + + +class HTTPServerThread(threading.Thread): + def __init__(self, use_ssl=False): + super().__init__() + self.host = 'localhost' + self.httpd = HTTPServer((self.host, 0), MockRequestHandler) + self.port = self.httpd.socket.getsockname()[1] + self.use_ssl = use_ssl + + if use_ssl: + ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ssl_context.options |= ssl.OP_NO_SSLv2 + ssl_context.verify_mode = ssl.CERT_NONE + ssl_context.load_cert_chain(os.path.join(TEST_DIR, 'server.crt'), + os.path.join(TEST_DIR, 'server.key')) + self.httpd.ssl_context = ssl_context + else: + self.httpd.ssl_context = None + + def run(self): + self.httpd.serve_forever() + + def shutdown(self): + self.httpd.shutdown() + self.httpd.server_close() + +def pytest_generate_tests(metafunc): + if not 'http_server' in metafunc.fixturenames: + return + + if getattr(metafunc.function, 'no_ssl', False): + params = ('plain',) + else: + params = ('plain', 'ssl') + + metafunc.parametrize("http_server", params, + indirect=True, scope='module') + +@pytest.fixture() +def http_server(request): + httpd = HTTPServerThread(use_ssl=(request.param == 'ssl')) + httpd.start() + request.addfinalizer(httpd.shutdown) + return httpd + +@pytest.fixture() +def conn(request, http_server): + if http_server.use_ssl: + ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ssl_context.options |= ssl.OP_NO_SSLv2 + ssl_context.verify_mode = ssl.CERT_REQUIRED + ssl_context.load_verify_locations(cafile=os.path.join(TEST_DIR, 'ca.crt')) + else: + ssl_context = None + conn = HTTPConnection(http_server.host, port=http_server.port, + ssl_context=ssl_context) + request.addfinalizer(conn.disconnect) + return conn + +@pytest.fixture(scope='module') +def random_fh(request): + fh = open('/dev/urandom', 'rb') + request.addfinalizer(fh.close) + return fh + +def check_http_connection(): + '''Skip test if we can't connect to ssl test server''' + + ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ssl_context.options |= ssl.OP_NO_SSLv2 + ssl_context.verify_mode = ssl.CERT_REQUIRED + ssl_context.set_default_verify_paths() + try: + conn = http.client.HTTPSConnection(SSL_TEST_HOST, context=ssl_context) + conn.request('GET', '/') + resp = conn.getresponse() + assert resp.status in (200, 301, 302) + except: + pytest.skip('%s not reachable but required for testing' % SSL_TEST_HOST) + finally: + conn.close() + +def test_connect_ssl(): + check_http_connection() + + ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ssl_context.options |= ssl.OP_NO_SSLv2 + ssl_context.verify_mode = ssl.CERT_REQUIRED + ssl_context.set_default_verify_paths() + + conn = HTTPConnection(SSL_TEST_HOST, ssl_context=ssl_context) + conn.send_request('GET', '/') + resp = conn.read_response() + assert resp.status in (200, 301, 302) + assert resp.path == '/' + conn.discard() + conn.disconnect() + +def test_invalid_ssl(): + check_http_connection() + + # Don't load certificates + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + context.options |= ssl.OP_NO_SSLv2 + context.verify_mode = ssl.CERT_REQUIRED + + conn = HTTPConnection(SSL_TEST_HOST, ssl_context=context) + with pytest.raises(ssl.SSLError): + conn.send_request('GET', '/') + conn.disconnect() + +def test_dns_one(monkeypatch): + monkeypatch.setattr(dugong, 'DNS_TEST_HOSTNAMES', + (('localhost', 80),)) + with pytest.raises(dugong.HostnameNotResolvable): + conn = HTTPConnection('foobar.invalid') + conn.connect() + +def test_dns_two(monkeypatch): + monkeypatch.setattr(dugong, 'DNS_TEST_HOSTNAMES', + (('grumpf.invalid', 80),)) + with pytest.raises(dugong.DNSUnavailable): + conn = HTTPConnection('foobar.invalid') + conn.connect() + +@pytest.mark.parametrize('test_port', (None, 8080)) +@pytest.mark.no_ssl +def test_http_proxy(http_server, monkeypatch, test_port): + test_host = 'www.foobarz.invalid' + test_path = '/someurl?barf' + + get_path = None + def do_GET(self): + nonlocal get_path + get_path = self.path + self.send_response(200) + self.send_header("Content-Type", 'application/octet-stream') + self.send_header("Content-Length", '0') + self.end_headers() + monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET) + + conn = HTTPConnection(test_host, test_port, + proxy=(http_server.host, http_server.port)) + try: + conn.send_request('GET', test_path) + resp = conn.read_response() + assert resp.status == 200 + conn.discard() + finally: + conn.disconnect() + + if test_port is None: + exp_path = 'http://%s%s' % (test_host, test_path) + else: + exp_path = 'http://%s:%d%s' % (test_host, test_port, test_path) + + assert get_path == exp_path + +class FakeSSLSocket: + def __init__(self, socket): + self.socket = socket + + def getpeercert(self): + return None + + def __getattr__(self, name): + return getattr(self.socket, name) + +class FakeSSLContext: + def wrap_socket(self, socket, server_hostname): + return FakeSSLSocket(socket) + def __bool__(self): + return True + +@pytest.mark.parametrize('test_port', (None, 8080)) +@pytest.mark.no_ssl +def test_connect_proxy(http_server, monkeypatch, test_port): + test_host = 'www.foobarz.invalid' + test_path = '/someurl?barf' + + connect_path = None + def do_CONNECT(self): + # Pretend we're the remote server too + nonlocal connect_path + connect_path = self.path + self.send_response(200) + self.end_headers() + self.close_connection = 0 + monkeypatch.setattr(MockRequestHandler, 'do_CONNECT', + do_CONNECT, raising=False) + + get_path = None + def do_GET(self): + nonlocal get_path + get_path = self.path + self.send_response(200) + self.send_header("Content-Type", 'application/octet-stream') + self.send_header("Content-Length", '0') + self.end_headers() + monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET) + + # We don't *actually* want to establish SSL, that'd be + # to complex for our mock server + monkeypatch.setattr('ssl.match_hostname', lambda x,y: True) + conn = HTTPConnection(test_host, test_port, + proxy=(http_server.host, http_server.port), + ssl_context=FakeSSLContext()) + try: + conn.send_request('GET', test_path) + resp = conn.read_response() + assert resp.status == 200 + conn.discard() + finally: + conn.disconnect() + + if test_port is None: + test_port = 443 + exp_path = '%s:%d' % (test_host, test_port) + assert connect_path == exp_path + assert get_path == test_path + +def get_chunked_GET_handler(path, chunks, delay=None): + '''Return GET handler for *path* sending *chunks* of data''' + + def do_GET(self): + if self.path != path: + self.send_error(500, 'Assertion failure: %s != %s' + % (self.path, path)) + return + self.send_response(200) + self.send_header("Content-Type", 'application/octet-stream') + self.send_header("Transfer-Encoding", 'chunked') + self.end_headers() + for (i, chunk_size) in enumerate(chunks): + if i % 3 == 0 and delay: + time.sleep(delay*1e-3) + self.wfile.write(('%x\r\n' % chunk_size).encode('us-ascii')) + if i % 3 == 1 and delay: + self.wfile.write(DUMMY_DATA[:chunk_size//2]) + time.sleep(delay*1e-3) + self.wfile.write(DUMMY_DATA[chunk_size//2:chunk_size]) + else: + self.wfile.write(DUMMY_DATA[:chunk_size]) + if i % 3 == 2 and delay: + time.sleep(delay*1e-3) + self.wfile.write(b'\r\n') + self.wfile.flush() + self.wfile.write(b'0\r\n\r\n') + return do_GET + +def test_get_pipeline(conn): + # We assume that internal buffers are big enough to hold + # a few requests + + paths = [ '/send_120_bytes' for _ in range(3) ] + + # Send requests + for path in paths: + crt = conn.co_send_request('GET', path) + for io_req in crt: + # If this fails, then internal buffers are too small + assert io_req.poll(10) + + # Read responses + for path in paths: + resp = conn.read_response() + assert resp.status == 200 + assert resp.path == path + assert conn.readall() == DUMMY_DATA[:120] + +def test_ssl_info(conn): + conn.get_ssl_cipher() + conn.get_ssl_peercert() + +def test_blocking_send(conn, random_fh, monkeypatch): + # Send requests until we block because all TCP buffers are full + + out_len = 102400 + in_len = 8192 + path = '/buo?com' + def do_GET(self): + self.rfile.read(in_len) + self.send_response(200) + self.send_header("Content-Type", 'application/octet-stream') + self.send_header("Content-Length", str(out_len)) + self.end_headers() + self.wfile.write(random_fh.read(out_len)) + monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET) + + for count in itertools.count(): + crt = conn.co_send_request('GET', path, body=random_fh.read(in_len)) + flag = False + for io_req in crt: + if not io_req.poll(1): + flag = True + break + if flag: + break + if count > 1000000: + pytest.fail("no blocking even after %d requests!?" % count) + + # Read responses + for i in range(count): + resp = conn.read_response() + assert resp.status == 200 + conn.discard() + + # Now we should be able to complete the request + assert io_req.poll(5) + with pytest.raises(StopIteration): + next(crt) + + resp = conn.read_response() + assert resp.status == 200 + conn.discard() + +def test_blocking_read(conn, monkeypatch): + path = '/foo/wurfl' + chunks = [120] * 10 + delay = 10 + + while True: + monkeypatch.setattr(MockRequestHandler, 'do_GET', + get_chunked_GET_handler(path, chunks, delay)) + conn.send_request('GET', path) + + resp = conn.read_response() + assert resp.status == 200 + + interrupted = 0 + parts = [] + while True: + crt = conn.co_read(100) + try: + while True: + io_req = next(crt) + interrupted += 1 + assert io_req.poll(5) + except StopIteration as exc: + buf = exc.value + if not buf: + break + parts.append(buf) + assert not conn.response_pending() + + assert _join(parts) == b''.join(DUMMY_DATA[:x] for x in chunks) + if interrupted >= 8: + break + elif delay > 5000: + pytest.fail('no blocking read even with %f sec sleep' % delay) + delay *= 2 + +def test_discard(conn, monkeypatch): + data_len = 512 + path = '/send_%d_bytes' % data_len + conn.send_request('GET', path) + resp = conn.read_response() + assert resp.status == 200 + assert resp.path == path + assert resp.length == data_len + conn.discard() + assert not conn.response_pending() + +def test_discard_chunked(conn, monkeypatch): + path = '/foo/wurfl' + chunks = [512, 312, 837, 361] + monkeypatch.setattr(MockRequestHandler, 'do_GET', + get_chunked_GET_handler(path, chunks)) + + conn.send_request('GET', path) + resp = conn.read_response() + assert resp.status == 200 + assert resp.path == path + assert resp.length is None + conn.discard() + assert not conn.response_pending() + +def test_read_text(conn): + conn.send_request('GET', '/send_%d_bytes' % len(DUMMY_DATA)) + conn.read_response() + fh = TextIOWrapper(conn) + assert fh.read() == DUMMY_DATA.decode('utf8') + assert not conn.response_pending() + +def test_read_text2(conn): + conn.send_request('GET', '/send_%d_bytes' % len(DUMMY_DATA)) + conn.read_response() + fh = TextIOWrapper(conn) + + # This used to fail because TextIOWrapper can't deal with bytearrays + fh.read(42) + +def test_read_text3(conn): + conn.send_request('GET', '/send_%d_bytes' % len(DUMMY_DATA)) + conn.read_response() + fh = TextIOWrapper(conn) + + # This used to fail because TextIOWrapper tries to read from + # the underlying fh even after getting '' + while True: + if not fh.read(77): + break + + assert not conn.response_pending() + +def test_read_identity(conn): + conn.send_request('GET', '/send_512_bytes') + resp = conn.read_response() + assert resp.status == 200 + assert resp.path == '/send_512_bytes' + assert resp.length == 512 + assert conn.readall() == DUMMY_DATA[:512] + assert not conn.response_pending() + +def test_conn_close_1(conn, monkeypatch): + # Regular read + data_size = 500 + conn._rbuf = dugong._Buffer(int(4/5*data_size)) + + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", 'application/octet-stream') + self.send_header("Connection", 'close') + self.end_headers() + self.wfile.write(DUMMY_DATA[:data_size]) + self.rfile.close() + self.wfile.close() + monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET) + + conn.send_request('GET', '/whatever') + resp = conn.read_response() + assert resp.status == 200 + assert conn.readall() == DUMMY_DATA[:data_size] + + with pytest.raises(ConnectionClosed): + conn.send_request('GET', '/whatever') + conn.read_response() + +def test_conn_close_2(conn, monkeypatch): + # Readinto + data_size = 500 + conn._rbuf = dugong._Buffer(int(4/5*data_size)) + + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", 'application/octet-stream') + self.send_header("Connection", 'close') + self.end_headers() + self.wfile.write(DUMMY_DATA[:data_size]) + self.rfile.close() + self.wfile.close() + monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET) + + conn.send_request('GET', '/whatever') + resp = conn.read_response() + assert resp.status == 200 + buf = memoryview(bytearray(2*data_size)) + got = conn.readinto(buf) + got += conn.readinto(buf[got:]) + assert got == data_size + assert buf[:data_size] == DUMMY_DATA[:data_size] + assert conn.readinto(buf[got:]) == 0 + + with pytest.raises(ConnectionClosed): + conn.send_request('GET', '/whatever') + conn.read_response() + +def test_conn_close_3(conn, monkeypatch): + # Server keeps reading + data_size = 500 + conn._rbuf = dugong._Buffer(int(4/5*data_size)) + + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", 'application/octet-stream') + self.send_header("Connection", 'close') + self.end_headers() + self.wfile.write(DUMMY_DATA[:data_size]) + self.wfile.close() + monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET) + + conn.send_request('GET', '/whatever') + resp = conn.read_response() + assert resp.status == 200 + assert conn.readall() == DUMMY_DATA[:data_size] + + conn.send_request('GET', '/whatever') + assert_raises(ConnectionClosed, conn.read_response) + +def test_conn_close_4(conn, monkeypatch): + # Content-Length should take precedence + data_size = 500 + conn._rbuf = dugong._Buffer(int(4/5*data_size)) + + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", 'application/octet-stream') + self.send_header("Content-Length", str(data_size)) + self.send_header("Connection", 'close') + self.end_headers() + self.wfile.write(DUMMY_DATA[:data_size+10]) + self.wfile.close() + monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET) + + conn.send_request('GET', '/whatever') + resp = conn.read_response() + assert resp.status == 200 + assert conn.readall() == DUMMY_DATA[:data_size] + +def test_conn_close_5(conn, monkeypatch): + # Pipelining + data_size = 500 + conn._rbuf = dugong._Buffer(int(4/5*data_size)) + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", 'application/octet-stream') + self.send_header("Content-Length", str(data_size)) + self.send_header("Connection", 'close') + self.end_headers() + self.wfile.write(DUMMY_DATA[:data_size]) + self.wfile.close() + self.rfile.close() + monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET) + + conn.send_request('GET', '/whatever_one') + conn.send_request('GET', '/whatever_two') + resp = conn.read_response() + assert resp.status == 200 + assert conn.readall() == DUMMY_DATA[:data_size] + assert_raises(dugong.ConnectionClosed, conn.read_response) + +@pytest.mark.no_ssl +def test_exhaust_buffer(conn): + conn._rbuf = dugong._Buffer(600) + conn.send_request('GET', '/send_512_bytes') + conn.read_response() + + # Test the case where the readbuffer is truncated and + # returned, instead of copied + conn._rbuf.compact() + for io_req in conn._co_fill_buffer(1): + io_req.poll() + assert conn._rbuf.b == 0 + assert conn._rbuf.e > 0 + buf = conn.read(600) + assert len(conn._rbuf.d) == 600 + assert buf == DUMMY_DATA[:len(buf)] + assert conn.readall() == DUMMY_DATA[len(buf):512] + +@pytest.mark.no_ssl +def test_full_buffer(conn): + conn._rbuf = dugong._Buffer(100) + conn.send_request('GET', '/send_512_bytes') + conn.read_response() + + buf = conn.read(101) + pos = len(buf) + assert buf == DUMMY_DATA[:pos] + + # Make buffer empty, but without capacity for more + assert conn._rbuf.e == 0 + conn._rbuf.e = len(conn._rbuf.d) + conn._rbuf.b = conn._rbuf.e + + assert conn.readall() == DUMMY_DATA[pos:512] + +def test_readinto_identity(conn): + conn.send_request('GET', '/send_512_bytes') + resp = conn.read_response() + assert resp.status == 200 + assert resp.path == '/send_512_bytes' + assert resp.length == 512 + parts = [] + while True: + buf = bytearray(600) + len_ = conn.readinto(buf) + if not len_: + break + parts.append(buf[:len_]) + assert _join(parts) == DUMMY_DATA[:512] + assert not conn.response_pending() + +def test_read_chunked(conn, monkeypatch): + path = '/foo/wurfl' + chunks = [300, 283, 377] + monkeypatch.setattr(MockRequestHandler, 'do_GET', + get_chunked_GET_handler(path, chunks)) + conn.send_request('GET', path) + resp = conn.read_response() + assert resp.status == 200 + assert resp.path == path + assert resp.length is None + assert conn.readall() == b''.join(DUMMY_DATA[:x] for x in chunks) + assert not conn.response_pending() + +def test_read_chunked2(conn, monkeypatch): + path = '/foo/wurfl' + chunks = [5] * 10 + monkeypatch.setattr(MockRequestHandler, 'do_GET', + get_chunked_GET_handler(path, chunks)) + conn.send_request('GET', path) + resp = conn.read_response() + assert resp.status == 200 + assert resp.length is None + assert resp.path == path + assert conn.readall() == b''.join(DUMMY_DATA[:x] for x in chunks) + assert not conn.response_pending() + +def test_readinto_chunked(conn, monkeypatch): + path = '/foo/wurfl' + chunks = [300, 317, 283] + monkeypatch.setattr(MockRequestHandler, 'do_GET', + get_chunked_GET_handler(path, chunks)) + conn.send_request('GET', path) + resp = conn.read_response() + assert resp.status == 200 + assert resp.length is None + assert resp.path == path + parts = [] + while True: + buf = bytearray(600) + len_ = conn.readinto(buf) + if not len_: + break + parts.append(buf[:len_]) + assert _join(parts) == b''.join(DUMMY_DATA[:x] for x in chunks) + assert not conn.response_pending() + +def test_double_read(conn): + conn.send_request('GET', '/send_10_bytes') + resp = conn.read_response() + assert resp.status == 200 + assert resp.length == 10 + assert resp.path == '/send_10_bytes' + with pytest.raises(dugong.StateError): + resp = conn.read_response() + +def test_read_raw(conn, monkeypatch): + path = '/ooops' + def do_GET(self): + assert self.path == path + self.send_response(200) + self.send_header("Content-Type", 'application/octet-stream') + self.end_headers() + self.wfile.write(b'body data') + self.wfile.close() + monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET) + conn.send_request('GET', path) + resp = conn.read_response() + assert resp.status == 200 + with pytest.raises(dugong.UnsupportedResponse): + conn.readall() + assert conn.read_raw(512) == b'body data' + assert conn.read_raw(512) == b'' + +def test_abort_read(conn, monkeypatch): + path = '/foo/wurfl' + chunks = [300, 317, 283] + monkeypatch.setattr(MockRequestHandler, 'do_GET', + get_chunked_GET_handler(path, chunks)) + conn.send_request('GET', path) + resp = conn.read_response() + assert resp.status == 200 + conn.read(200) + conn.disconnect() + assert_raises(dugong.ConnectionClosed, conn.read, 200) + +def test_abort_co_read(conn, monkeypatch): + # We need to delay the write to ensure that we encounter a blocking read + path = '/foo/wurfl' + chunks = [300, 317, 283] + delay = 10 + while True: + monkeypatch.setattr(MockRequestHandler, 'do_GET', + get_chunked_GET_handler(path, chunks, delay)) + conn.send_request('GET', path) + resp = conn.read_response() + assert resp.status == 200 + cofun = conn.co_read(450) + try: + next(cofun) + except StopIteration: + # Not good, need to wait longer + pass + else: + break + finally: + conn.disconnect() + + if delay > 5000: + pytest.fail('no blocking read even with %f sec sleep' % delay) + delay *= 2 + + assert_raises(dugong.ConnectionClosed, next, cofun) + +def test_abort_write(conn): + conn.send_request('PUT', '/allgood', body=BodyFollowing(42)) + conn.write(b'fooo') + conn.disconnect() + assert_raises(dugong.ConnectionClosed, conn.write, b'baar') + +def test_write_toomuch(conn): + conn.send_request('PUT', '/allgood', body=BodyFollowing(42)) + with pytest.raises(dugong.ExcessBodyData): + conn.write(DUMMY_DATA[:43]) + +def test_write_toolittle(conn): + conn.send_request('PUT', '/allgood', body=BodyFollowing(42)) + conn.write(DUMMY_DATA[:24]) + with pytest.raises(dugong.StateError): + conn.send_request('GET', '/send_5_bytes') + +def test_write_toolittle2(conn): + conn.send_request('PUT', '/allgood', body=BodyFollowing(42)) + conn.write(DUMMY_DATA[:24]) + with pytest.raises(dugong.StateError): + conn.read_response() + +def test_write_toolittle3(conn): + conn.send_request('GET', '/send_10_bytes') + conn.send_request('PUT', '/allgood', body=BodyFollowing(42)) + conn.write(DUMMY_DATA[:24]) + resp = conn.read_response() + assert resp.status == 200 + assert resp.path == '/send_10_bytes' + assert len(conn.readall()) == 10 + with pytest.raises(dugong.StateError): + conn.read_response() + +def test_put(conn): + data = DUMMY_DATA + conn.send_request('PUT', '/allgood', body=data) + resp = conn.read_response() + conn.discard() + assert resp.status == 204 + assert resp.length == 0 + assert resp.reason == 'MD5 matched' + + headers = CaseInsensitiveDict() + headers['Content-MD5'] = 'nUzaJEag3tOdobQVU/39GA==' + conn.send_request('PUT', '/allgood', body=data, headers=headers) + resp = conn.read_response() + conn.discard() + assert resp.status == 400 + assert resp.reason.startswith('MD5 mismatch') + +def test_put_separate(conn): + data = DUMMY_DATA + conn.send_request('PUT', '/allgood', body=BodyFollowing(len(data))) + conn.write(data) + resp = conn.read_response() + conn.discard() + assert resp.status == 204 + assert resp.length == 0 + assert resp.reason == 'Ok, but no MD5' + + headers = CaseInsensitiveDict() + headers['Content-MD5'] = b64encode(hashlib.md5(data).digest()).decode('ascii') + conn.send_request('PUT', '/allgood', body=BodyFollowing(len(data)), + headers=headers) + conn.write(data) + resp = conn.read_response() + conn.discard() + assert resp.status == 204 + assert resp.length == 0 + assert resp.reason == 'MD5 matched' + + headers['Content-MD5'] = 'nUzaJEag3tOdobQVU/39GA==' + conn.send_request('PUT', '/allgood', body=BodyFollowing(len(data)), + headers=headers) + conn.write(data) + resp = conn.read_response() + conn.discard() + assert resp.status == 400 + assert resp.reason.startswith('MD5 mismatch') + +def test_100cont(conn, monkeypatch): + + path = '/check_this_out' + def handle_expect_100(self): + if self.path != path: + self.send_error(500, 'Assertion error, %s != %s' + % (self.path, path)) + else: + self.send_error(403) + monkeypatch.setattr(MockRequestHandler, 'handle_expect_100', + handle_expect_100) + + conn.send_request('PUT', path, body=BodyFollowing(256), + expect100=True) + resp = conn.read_response() + assert resp.status == 403 + conn.discard() + + def handle_expect_100(self): + if self.path != path: + self.send_error(500, 'Assertion error, %s != %s' + % (self.path, path)) + return + + self.send_response_only(100) + self.end_headers() + return True + monkeypatch.setattr(MockRequestHandler, 'handle_expect_100', + handle_expect_100) + conn.send_request('PUT', path, body=BodyFollowing(256), expect100=True) + resp = conn.read_response() + assert resp.status == 100 + assert resp.length == 0 + conn.write(DUMMY_DATA[:256]) + resp = conn.read_response() + assert resp.status == 204 + assert resp.length == 0 + +def test_100cont_2(conn, monkeypatch): + def handle_expect_100(self): + self.send_error(403) + monkeypatch.setattr(MockRequestHandler, 'handle_expect_100', + handle_expect_100) + conn.send_request('PUT', '/fail_with_403', body=BodyFollowing(256), + expect100=True) + + with pytest.raises(dugong.StateError): + conn.send_request('PUT', '/fail_with_403', body=BodyFollowing(256), expect100=True) + + conn.read_response() + conn.readall() + +def test_100cont_3(conn, monkeypatch): + def handle_expect_100(self): + self.send_error(403) + monkeypatch.setattr(MockRequestHandler, 'handle_expect_100', + handle_expect_100) + conn.send_request('PUT', '/fail_with_403', body=BodyFollowing(256), expect100=True) + + with pytest.raises(dugong.StateError): + conn.write(b'barf!') + + conn.read_response() + conn.readall() + +def test_aborted_write1(conn, monkeypatch, random_fh): + BUFSIZE = 64*1024 + + # monkeypatch request handler + def do_PUT(self): + # Read half the data, then generate error and + # close connection + self.rfile.read(BUFSIZE) + self.send_error(code=401, message='Please stop!') + self.close_connection = True + monkeypatch.setattr(MockRequestHandler, 'do_PUT', do_PUT) + + # Send request + conn.send_request('PUT', '/big_object', body=BodyFollowing(BUFSIZE*50), + expect100=True) + resp = conn.read_response() + assert resp.status == 100 + assert resp.length == 0 + + # Try to write data + with pytest.raises(ConnectionClosed): + for _ in range(50): + conn.write(random_fh.read(BUFSIZE)) + + # Nevertheless, try to read response + resp = conn.read_response() + assert resp.status == 401 + assert resp.reason == 'Please stop!' + +def test_aborted_write2(conn, monkeypatch, random_fh): + BUFSIZE = 64*1024 + + # monkeypatch request handler + def do_PUT(self): + # Read half the data, then silently close connection + self.rfile.read(BUFSIZE) + self.close_connection = True + monkeypatch.setattr(MockRequestHandler, 'do_PUT', do_PUT) + + # Send request + conn.send_request('PUT', '/big_object', body=BodyFollowing(BUFSIZE*50), + expect100=True) + resp = conn.read_response() + assert resp.status == 100 + assert resp.length == 0 + + # Try to write data + with pytest.raises(ConnectionClosed): + for _ in range(50): + conn.write(random_fh.read(BUFSIZE)) + + # Nevertheless, try to read response + assert_raises(ConnectionClosed, conn.read_response) + +def test_read_toomuch(conn): + conn.send_request('GET', '/send_10_bytes') + conn.send_request('GET', '/send_8_bytes') + resp = conn.read_response() + assert resp.status == 200 + assert resp.path == '/send_10_bytes' + assert conn.readall() == DUMMY_DATA[:10] + assert conn.read(8) == b'' + +def test_read_toolittle(conn): + conn.send_request('GET', '/send_10_bytes') + resp = conn.read_response() + assert resp.status == 200 + assert resp.path == '/send_10_bytes' + buf = conn.read(8) + assert buf == DUMMY_DATA[:len(buf)] + with pytest.raises(dugong.StateError): + resp = conn.read_response() + +def test_empty_response(conn): + conn.send_request('HEAD', '/send_512_bytes') + resp = conn.read_response() + assert resp.status == 200 + assert resp.path == '/send_512_bytes' + assert resp.length == 0 + + # Check that we can go to the next response without + # reading anything + assert not conn.response_pending() + conn.send_request('GET', '/send_512_bytes') + resp = conn.read_response() + assert resp.status == 200 + assert resp.path == '/send_512_bytes' + assert resp.length == 512 + assert conn.readall() == DUMMY_DATA[:512] + assert not conn.response_pending() + +def test_head(conn, monkeypatch): + conn.send_request('HEAD', '/send_10_bytes') + resp = conn.read_response() + assert resp.status == 200 + assert len(conn.readall()) == 0 + + def do_HEAD(self): + self.send_error(317) + monkeypatch.setattr(MockRequestHandler, 'do_HEAD', do_HEAD) + conn.send_request('HEAD', '/fail_with_317') + resp = conn.read_response() + assert resp.status == 317 + assert len(conn.readall()) == 0 + +@pytest.fixture(params=(63,64,65,100,99,103,500,511,512,513)) +def buffer_size(request): + return request.param + +def test_smallbuffer(conn, buffer_size): + conn._rbuf = dugong._Buffer(buffer_size) + conn.send_request('GET', '/send_512_bytes') + resp = conn.read_response() + assert resp.status == 200 + assert resp.path == '/send_512_bytes' + assert resp.length == 512 + assert conn.readall() == DUMMY_DATA[:512] + assert not conn.response_pending() + +def test_mutable_read(conn): + # Read data and modify it, to make sure that this doesn't + # affect the buffer + + conn._rbuf = dugong._Buffer(129) + conn.send_request('GET', '/send_512_bytes') + conn.read_response() + + # Assert that buffer is full, but does not start at beginning + assert conn._rbuf.b > 0 + + # Need to avoid conn.read(), because it converts to bytes + buf = dugong.eval_coroutine(conn.co_read(150)) + pos = len(buf) + assert buf == DUMMY_DATA[:pos] + memoryview(buf)[:10] = b'\0' * 10 + + # Assert that buffer is empty + assert conn._rbuf.b == 0 + assert conn._rbuf.e == 0 + buf = dugong.eval_coroutine(conn.co_read(150)) + assert buf == DUMMY_DATA[pos:pos+len(buf)] + memoryview(buf)[:10] = b'\0' * 10 + pos += len(buf) + + assert conn.readall() == DUMMY_DATA[pos:512] + assert not conn.response_pending() + +def test_recv_timeout(conn, monkeypatch): + conn.timeout = 1 + + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", 'application/octet-stream') + self.send_header("Content-Length", '50') + self.end_headers() + self.wfile.write(b'x' * 25) + self.wfile.flush() + monkeypatch.setattr(MockRequestHandler, 'do_GET', do_GET) + + conn.send_request('GET', '/send_something') + resp = conn.read_response() + assert resp.status == 200 + assert conn.read(50) == b'x' * 25 + assert_raises(dugong.ConnectionTimedOut, conn.read, 50) + +def test_send_timeout(conn, monkeypatch, random_fh): + conn.timeout = 1 + + def do_PUT(self): + # Read just a tiny bit + self.rfile.read(256) + + # We need to sleep, or the rest of the incoming data will + # be parsed as the next request. + time.sleep(2*conn.timeout) + monkeypatch.setattr(MockRequestHandler, 'do_PUT', do_PUT) + + # We don't know how much data can be buffered, so we + # claim to send a lot and do so in a loop. + len_ = 1024**3 + conn.send_request('PUT', '/recv_something', body=BodyFollowing(len_)) + with pytest.raises(dugong.ConnectionTimedOut): + while len_ > 0: + conn.write(random_fh.read(min(len_, 16*1024))) + + +DUMMY_DATA = ','.join(str(x) for x in range(10000)).encode() + +class MockRequestHandler(BaseHTTPRequestHandler): + + server_version = "MockHTTP" + protocol_version = 'HTTP/1.1' + + def log_message(self, format, *args): + pass + + def handle(self): + # Ignore exceptions resulting from the client closing + # the connection. + try: + return super().handle() + except ValueError as exc: + if exc.args == ('I/O operation on closed file.',): + pass + else: + raise + # Linux generates BrokenPipeError, FreeBSD uses ConnectionResetError + except (BrokenPipeError, ConnectionResetError): + pass + + def do_GET(self): + len_ = int(self.headers['Content-Length']) + if len_: + self.rfile.read(len_) + + hit = re.match(r'^/send_([0-9]+)_bytes', self.path) + if hit: + len_ = int(hit.group(1)) + self.do_HEAD() + self.wfile.write(DUMMY_DATA[:len_]) + return + + self.send_error(500) + + def do_PUT(self): + encoding = self.headers['Content-Encoding'] + if encoding and encoding != 'identity': + self.send_error(415) + return + + len_ = int(self.headers['Content-Length']) + data = self.rfile.read(len_) + if 'Content-MD5' in self.headers: + md5 = b64encode(hashlib.md5(data).digest()).decode('ascii') + if md5 != self.headers['Content-MD5']: + self.send_error(400, 'MD5 mismatch: %s vs %s' + % (md5, self.headers['Content-MD5'])) + return + + self.send_response(204, 'MD5 matched') + else: + self.send_response(204, 'Ok, but no MD5') + + self.send_header('Content-Length', '0') + self.end_headers() + + def do_HEAD(self): + hit = re.match(r'^/send_([0-9]+)_bytes', self.path) + if hit: + len_ = int(hit.group(1)) + self.send_response(200) + self.send_header("Content-Type", 'application/octet-stream') + self.send_header("Content-Length", str(len_)) + self.end_headers() + return + + # No idea + self.send_error(500) + + def send_error(self, code, message=None): + # Overwritten to not close connection on errors and provide + # content-length + try: + shortmsg, longmsg = self.responses[code] + except KeyError: + shortmsg, longmsg = '???', '???' + if message is None: + message = shortmsg + explain = longmsg + self.log_error("code %d, message %s", code, message) + # using _quote_html to prevent Cross Site Scripting attacks (see bug #1100201) + content = (self.error_message_format % {'code': code, 'message': _quote_html(message), + 'explain': explain}).encode('utf-8', 'replace') + self.send_response(code, message) + self.send_header("Content-Type", self.error_content_type) + self.send_header("Content-Length", str(len(content))) + self.end_headers() + if self.command != 'HEAD' and code >= 200 and code not in (204, 304): + self.wfile.write(content) diff --git a/test/test_examples.py b/test/test_examples.py new file mode 100755 index 0000000..41dfe8b --- /dev/null +++ b/test/test_examples.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +''' +test_dugong.py - Unit tests for Dugong - run with py.test + +Copyright © 2014 Nikolaus Rath + +This module may be distributed under the terms of the Python Software Foundation +License Version 2. The complete license text may be retrieved from +http://hg.python.org/cpython/file/65f2c92ed079/LICENSE. +''' + +if __name__ == '__main__': + import pytest + import sys + sys.exit(pytest.main([__file__] + sys.argv[1:])) + +import subprocess +import os +import sys +import pytest +import threading +from http.server import SimpleHTTPRequestHandler +from socketserver import TCPServer + +try: + import asyncio +except ImportError: + asyncio = None + +basename = os.path.join(os.path.dirname(__file__), '..') + +class HTTPRequestHandler(SimpleHTTPRequestHandler): + protocol_version = "HTTP/1.1" + +class HTTPServerThread(threading.Thread): + def __init__(self): + super().__init__() + self.host = 'localhost' + self.httpd = TCPServer((self.host, 0), HTTPRequestHandler) + self.port = self.httpd.socket.getsockname()[1] + self.url = 'http://%s:%d' % (self.host, self.port) + + def run(self): + self.httpd.serve_forever() + + def shutdown(self): + self.httpd.shutdown() + self.httpd.server_close() + +@pytest.fixture(scope='module') +def mock_server(request): + os.chdir(basename) + httpd = HTTPServerThread() + httpd.start() + request.addfinalizer(httpd.shutdown) + return httpd + +def test_httpcat(mock_server): + cmdline = [sys.executable, 'examples/httpcat.py', + mock_server.url + '/setup.py' ] + with open('/dev/null', 'wb') as devnull: + subprocess.check_call(cmdline, stdout=devnull) + +def test_extract_links(mock_server): + cmdline = [sys.executable, 'examples/extract_links.py', + mock_server.url + '/test/' ] + with open('/dev/null', 'wb') as devnull: + subprocess.check_call(cmdline, stdout=devnull) + +@pytest.mark.skipif(asyncio is None, + reason='asyncio module not available') +def test_pipeline1(mock_server): + cmdline = [sys.executable, 'examples/pipeline1.py' ] + for name in os.listdir('test'): + if os.path.isdir('test/'+name): + name += '/' # avoid redirect + cmdline.append('%s/test/%s' % (mock_server.url, name)) + + with open('/dev/null', 'wb') as devnull: + subprocess.check_call(cmdline, stdout=devnull) -- cgit v1.2.3 From e3b9dffb09a200f027e76ba36ba7cd129bc09c93 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Fri, 7 Oct 2016 20:44:21 -0700 Subject: Import python-dugong_3.7+dfsg-3.debian.tar.xz [dgit import tarball python-dugong 3.7+dfsg-3 python-dugong_3.7+dfsg-3.debian.tar.xz] --- .git-dpm | 11 +++++ README.source | 16 +++++++ changelog | 85 ++++++++++++++++++++++++++++++++++++ compat | 1 + control | 61 ++++++++++++++++++++++++++ copyright | 62 ++++++++++++++++++++++++++ patches/series | 1 + patches/use-local-intersphinx.patch | 28 ++++++++++++ python-dugong-doc.doc-base | 8 ++++ python-dugong-doc.docs | 2 + python.inv | Bin 0 -> 124722 bytes rules | 35 +++++++++++++++ source/format | 1 + source/include-binaries | 2 + source/options | 4 ++ tests/control | 2 + tests/upstream-standard | 3 ++ upstream-signing-key.pgp | Bin 0 -> 2722 bytes watch | 8 ++++ 19 files changed, 330 insertions(+) create mode 100644 .git-dpm create mode 100644 README.source create mode 100644 changelog create mode 100644 compat create mode 100644 control create mode 100644 copyright create mode 100644 patches/series create mode 100644 patches/use-local-intersphinx.patch create mode 100644 python-dugong-doc.doc-base create mode 100644 python-dugong-doc.docs create mode 100644 python.inv create mode 100755 rules create mode 100644 source/format create mode 100644 source/include-binaries create mode 100644 source/options create mode 100644 tests/control create mode 100755 tests/upstream-standard create mode 100644 upstream-signing-key.pgp create mode 100644 watch diff --git a/.git-dpm b/.git-dpm new file mode 100644 index 0000000..fac2792 --- /dev/null +++ b/.git-dpm @@ -0,0 +1,11 @@ +# see git-dpm(1) from git-dpm package +cdf2dfcb0896376f9ec68c488bdad9fbd208f895 +cdf2dfcb0896376f9ec68c488bdad9fbd208f895 +4d86a28f722ed797e6780a871eeadc5b80c5686f +4d86a28f722ed797e6780a871eeadc5b80c5686f +python-dugong_3.7+dfsg.orig.tar.bz2 +44efbf4180d73c2d1715b53bcc89643a8fc1ce28 +51028 +debianTag="debian/%e%v" +patchedTag="patched/%e%v" +upstreamTag="upstream/%e%u" diff --git a/README.source b/README.source new file mode 100644 index 0000000..1e5bcce --- /dev/null +++ b/README.source @@ -0,0 +1,16 @@ + +* As requested by ftpmaster, the upstream tarball is repacked to + remove minified javascript files. + +* Since removing just the minified javascript files would leave the + rest of them broken, we are removing the entire pre-rendered HTML + documentation. There is no loss of information, since the entire + documentation (including the problematic javascript files) is + regenerated during build. + +* Repackaging of the upstream tarball is done with uscan. The list of + excluded files is specified in the Files-Excluded paragraph in + debian/copyright. To retrieve and repack the latest upstream + tarball, run + + # uscan --repack diff --git a/changelog b/changelog new file mode 100644 index 0000000..65401be --- /dev/null +++ b/changelog @@ -0,0 +1,85 @@ +python-dugong (3.7+dfsg-3) unstable; urgency=medium + + * Added missing python3-pytest-catchlog dependency for tests. + + -- Nikolaus Rath Fri, 07 Oct 2016 20:44:21 -0700 + +python-dugong (3.7+dfsg-2) unstable; urgency=medium + + * Fixed py.test invocation during build. + + -- Nikolaus Rath Mon, 20 Jun 2016 11:54:57 -0700 + +python-dugong (3.7+dfsg-1) unstable; urgency=medium + + * new upstream version + + -- Nikolaus Rath Mon, 20 Jun 2016 11:01:26 -0700 + +python-dugong (3.5+dfsg-2) UNRELEASED; urgency=medium + + * Fixed VCS URL (https) + + -- Ondřej Nový Tue, 29 Mar 2016 22:01:44 +0200 + +python-dugong (3.5+dfsg-1) unstable; urgency=medium + + * New upstream release. + + -- Nikolaus Rath Sat, 31 Jan 2015 20:12:05 -0800 + +python-dugong (3.4+dfsg-1) experimental; urgency=medium + + * New upstream release. + * Dropped fix_test_examples.diff, fixed upstream. + * Bumped Standard-Version to 3.9.6 (no changes required). + + -- Nikolaus Rath Sat, 29 Nov 2014 13:34:00 -0800 + +python-dugong (3.3+dfsg-2) unstable; urgency=medium + + * Skip unit tests in test/test_examples.py. These tests require network + connectivity, and in addition to that the hardcoded remote server no + longer works (thus causing the tests to fail even if there is + connectivity). Closes: #769250 + + -- Nikolaus Rath Fri, 14 Nov 2014 15:20:44 -0800 + +python-dugong (3.3+dfsg-1) unstable; urgency=medium + + * New upstream release. + + -- Nikolaus Rath Wed, 06 Aug 2014 18:45:37 -0700 + +python-dugong (3.2+dfsg-1) unstable; urgency=medium + + * Removed some packaging workarounds for Python 3.3 that are no longer + required. + * New upstream release. + * Dropped patches, included upstream: + - disable_mock_server_logging.diff + - fix_test_abort_co_read.diff + + -- Nikolaus Rath Sun, 27 Jul 2014 17:06:09 -0700 + +python-dugong (3.1+dfsg-1) unstable; urgency=medium + + * New upstream release. + * Added disable_mock_server_logging.diff to make adt-run happy. + * Added fix_test_abort_co_read.diff, cherry-picked from upstream to fix a + unit test failure. + + -- Nikolaus Rath Sat, 28 Jun 2014 13:13:56 -0700 + +python-dugong (3.0+dfsg-1) unstable; urgency=medium + + * New upstream release. + * Add kFreeBSD support (Closes: #745243) + + -- Nikolaus Rath Sun, 20 Apr 2014 17:25:22 -0700 + +python-dugong (2.2+dfsg-1) unstable; urgency=medium + + * First official debian release. Closes: 741387 + + -- Nikolaus Rath Thu, 27 Mar 2014 16:41:30 -0700 diff --git a/compat b/compat new file mode 100644 index 0000000..ec63514 --- /dev/null +++ b/compat @@ -0,0 +1 @@ +9 diff --git a/control b/control new file mode 100644 index 0000000..eeb696a --- /dev/null +++ b/control @@ -0,0 +1,61 @@ +Source: python-dugong +Section: python +X-Python3-Version: >= 3.4 +Priority: optional +Uploaders: Debian Python Modules Team +Maintainer: Nikolaus Rath +Build-Depends: debhelper (>= 9), + python3-all, + python3-sphinx (>= 1.0.7+dfsg), + python3-pytest (>= 2.8.0), + python3-pytest-catchlog, + dh-python, + python3-setuptools +Standards-Version: 3.9.6 +XS-Testsuite: autopkgtest +Homepage: https://bitbucket.org/nikratio/python-dugong +Vcs-Git: https://anonscm.debian.org/git/python-modules/packages/python-dugong.git +Vcs-Browser: https://anonscm.debian.org/cgit/python-modules/packages/python-dugong.git + +Package: python3-dugong +Architecture: all +Depends: ${misc:Depends}, + ${python3:Depends} +Provides: ${python3:Provides} +Suggests: python-dugong-doc +Description: HTTP 1.1 client module for Python + The Python Dugong module provides an API for communicating with HTTP 1.1 + servers. It is an alternative to the standard library's http.client (formerly + httplib) module. In contrast to http.client, Dugong: + . + * allows you to send multiple requests right after each other without having + to read the responses first. + * supports waiting for 100-continue before sending the request body. + * raises an exception instead of silently delivering partial data if the + connection is closed before all data has been received. + * raises one specific exception (ConnectionClosed) if the connection has been + closed (while http.client connection may raise any of BrokenPipeError, + BadStatusLine, ConnectionAbortedError, ConnectionResetError, IncompleteRead + or simply return '' on read) + * supports non-blocking, asynchronous operation and is compatible with the + asyncio module. + * can in most cases distinguish between an unavailable DNS server and + an unresolvable hostname. + * is not compatible with old HTTP 0.9 or 1.0 servers. + . + All request and response headers are represented as str, but must be encodable + in latin1. Request and response body must be bytes-like objects or binary + streams. + +Package: python-dugong-doc +Architecture: all +Section: doc +Recommends: python3-dugong +Depends: ${sphinxdoc:Depends}, + ${misc:Depends} +Description: HTTP 1.1 client module for Python (documentation) + The Python Dugong module provides an API for communicating with HTTP 1.1 + servers. It is an alternative to the standard library's http.client (formerly + httplib) module. + . + This package provides the documentation. diff --git a/copyright b/copyright new file mode 100644 index 0000000..ee32e84 --- /dev/null +++ b/copyright @@ -0,0 +1,62 @@ +Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ +Upstream-Name: python-dugong +Upstream-Contact: Nikolaus Rath +Source: https://bitbucket.org/nikratio/python-dugong/src +Files-Excluded: doc/html + +Files: * +Copyright: Copyright 2013-2014 Nikolaus Rath +License: PSF + 1. This LICENSE AGREEMENT is between the Python Software Foundation + ("PSF"), and the Individual or Organization ("Licensee") accessing and + otherwise using this software ("Python") in source or binary form and + its associated documentation. + . + 2. Subject to the terms and conditions of this License Agreement, PSF hereby + grants Licensee a nonexclusive, royalty-free, world-wide license to + reproduce, analyze, test, perform and/or display publicly, prepare + derivative works, distribute, and otherwise use Python alone or in + any derivative version, provided, however, that PSF's License + Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, + 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012, + 2013, 2014 Python Software Foundation; All Rights Reserved" are + retained in Python alone or in any derivative version prepared by + Licensee. + . + 3. In the event Licensee prepares a derivative work that is based on + or incorporates Python or any part thereof, and wants to make + the derivative work available to others as provided herein, then + Licensee hereby agrees to include in any such work a brief summary of + the changes made to Python. + . + 4. PSF is making Python available to Licensee on an "AS IS" + basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR + IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND + DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS + FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT + INFRINGE ANY THIRD PARTY RIGHTS. + . + 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON + FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS + A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, + OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + . + 6. This License Agreement will automatically terminate upon a material + breach of its terms and conditions. + . + 7. Nothing in this License Agreement shall be deemed to create any + relationship of agency, partnership, or joint venture between PSF and + Licensee. This License Agreement does not grant permission to use PSF + trademarks or trade name in a trademark sense to endorse or promote + products or services of Licensee, or any third party. + . + 8. By copying, installing or otherwise using Python, Licensee + agrees to be bound by the terms and conditions of this License + Agreement. + +Files: dugong/__init__.py +Comment: This applies to the CaseInsensitiveDict class implementation. +Copyright: Copyright 2013 Kenneth Reitz +License: Apache + On Debian systems the full text of the Apache 2.0 license can be + found in the `/usr/share/common-licenses/Apache-2.0' file. diff --git a/patches/series b/patches/series new file mode 100644 index 0000000..6fea815 --- /dev/null +++ b/patches/series @@ -0,0 +1 @@ +use-local-intersphinx.patch diff --git a/patches/use-local-intersphinx.patch b/patches/use-local-intersphinx.patch new file mode 100644 index 0000000..5b7de44 --- /dev/null +++ b/patches/use-local-intersphinx.patch @@ -0,0 +1,28 @@ +From cdf2dfcb0896376f9ec68c488bdad9fbd208f895 Mon Sep 17 00:00:00 2001 +From: Nikolaus Rath +Date: Thu, 8 Oct 2015 11:58:34 -0700 +Subject: Use local intersphinx inventory + +Forwarded: not-needed +Last-Update: 2014-03-11 + Instead of downloading the Python intersphinx directory + at build time, use the cached copy shipped in debian/. +Patch-Name: use-local-intersphinx.patch +--- + rst/conf.py | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/rst/conf.py b/rst/conf.py +index f312..851f 100644 +--- a/rst/conf.py ++++ b/rst/conf.py +@@ -8,7 +8,8 @@ import os.path + sys.path.append(os.path.abspath('..')) + + extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx' ] +-intersphinx_mapping = {'python': ('http://docs.python.org/3/', None) } ++intersphinx_mapping = {'python': ('http://docs.python.org/3/', ++ '../debian/python.inv')} + templates_path = ['_templates'] + source_suffix = '.rst' + source_encoding = 'utf-8' diff --git a/python-dugong-doc.doc-base b/python-dugong-doc.doc-base new file mode 100644 index 0000000..5a62e28 --- /dev/null +++ b/python-dugong-doc.doc-base @@ -0,0 +1,8 @@ +Document: python-dugong +Title: Python-Dugong API Documentation +Author: Nikolaus Rath +Section: Programming/Python + +Format: HTML +Index: /usr/share/doc/python-dugong-doc/html/index.html +Files: /usr/share/doc/python-dugong-doc/html/*.html diff --git a/python-dugong-doc.docs b/python-dugong-doc.docs new file mode 100644 index 0000000..1eefda5 --- /dev/null +++ b/python-dugong-doc.docs @@ -0,0 +1,2 @@ +doc/html/ +examples/ diff --git a/python.inv b/python.inv new file mode 100644 index 0000000..6a277e8 Binary files /dev/null and b/python.inv differ diff --git a/rules b/rules new file mode 100755 index 0000000..7fb852b --- /dev/null +++ b/rules @@ -0,0 +1,35 @@ +#!/usr/bin/make -f +# -*- makefile -*- + +#export DH_VERBOSE=1 +export PYBUILD_NAME=dugong +export PYBUILD_TEST_PYTEST=1 +export PYBUILD_TEST_ARGS=--installed "{dir}/test/" + +%: + dh $@ --with python3,sphinxdoc --buildsystem=pybuild + +.PHONY: override_dh_auto_build +override_dh_auto_build: + dh_auto_build + + # Build docs + python3 setup.py build_sphinx + +.PHONY: override_dh_auto_clean +override_dh_auto_clean: + dh_auto_clean + rm -rf doc/html doc/doctrees + +.PHONY: update_intersphinx +update_intersphinx: + wget http://docs.python.org/3/objects.inv -O debian/python.inv + +.PHONY: get-orig-source +get-orig-source: + uscan --rename --destdir=$(CURDIR)/.. --repack --force-download \ + --download-current-version --compression xz + +.PHONY: uscan +uscan: + uscan --rename --destdir=$(CURDIR)/.. --repack --compression xz diff --git a/source/format b/source/format new file mode 100644 index 0000000..163aaf8 --- /dev/null +++ b/source/format @@ -0,0 +1 @@ +3.0 (quilt) diff --git a/source/include-binaries b/source/include-binaries new file mode 100644 index 0000000..fb217db --- /dev/null +++ b/source/include-binaries @@ -0,0 +1,2 @@ +debian/python.inv +debian/upstream-signing-key.pgp diff --git a/source/options b/source/options new file mode 100644 index 0000000..c5f99fb --- /dev/null +++ b/source/options @@ -0,0 +1,4 @@ +compression = "xz" +compression-level = 6 +extend-diff-ignore = "\.pyc$" +tar-ignore = "*.pyc" diff --git a/tests/control b/tests/control new file mode 100644 index 0000000..4ada917 --- /dev/null +++ b/tests/control @@ -0,0 +1,2 @@ +Tests: upstream-standard +Depends: python3-pytest, python3-dugong, python3-pytest-catchlog diff --git a/tests/upstream-standard b/tests/upstream-standard new file mode 100755 index 0000000..ded21b9 --- /dev/null +++ b/tests/upstream-standard @@ -0,0 +1,3 @@ +#!/bin/sh + +exec py.test-3 --installed test/ diff --git a/upstream-signing-key.pgp b/upstream-signing-key.pgp new file mode 100644 index 0000000..d2ea347 Binary files /dev/null and b/upstream-signing-key.pgp differ diff --git a/watch b/watch new file mode 100644 index 0000000..170ac3f --- /dev/null +++ b/watch @@ -0,0 +1,8 @@ +# watch control file for uscan +version=3 +opts=\ +pgpsigurlmangle=s/$/.asc/,\ +uversionmangle=s/$/+dfsg/,\ + https://pypi.python.org/pypi/dugong/ \ + .*/dugong-(\d\S*)\.tar\.bz2 \ + debian -- cgit v1.2.3 From 98ee4f50e9b178c82675ce3af153af433b9713c8 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Thu, 8 Oct 2015 11:58:34 -0700 Subject: Use local intersphinx inventory Forwarded: not-needed Last-Update: 2014-03-11 Instead of downloading the Python intersphinx directory at build time, use the cached copy shipped in debian/. Patch-Name: use-local-intersphinx.patch Gbp-Pq: Name use-local-intersphinx.patch --- rst/conf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rst/conf.py b/rst/conf.py index f312145..851f20c 100644 --- a/rst/conf.py +++ b/rst/conf.py @@ -8,7 +8,8 @@ import os.path sys.path.append(os.path.abspath('..')) extensions = ['sphinx.ext.autodoc', 'sphinx.ext.intersphinx' ] -intersphinx_mapping = {'python': ('http://docs.python.org/3/', None) } +intersphinx_mapping = {'python': ('http://docs.python.org/3/', + '../debian/python.inv')} templates_path = ['_templates'] source_suffix = '.rst' source_encoding = 'utf-8' -- cgit v1.2.3