summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAndrej Shadura <andrewsh@debian.org>2018-12-21 22:55:20 +0100
committerAndrej Shadura <andrewsh@debian.org>2018-12-21 22:55:20 +0100
commit9cb59f187cc7f474669b46471188fc7adc4801dd (patch)
tree20998edc5ef256d4ad9486e389b57752c68e4c29
parent6e939c05e1b5ed4f6fe85c4933eebf62a6e822f5 (diff)
parent0eeb27df39ef880188c3eca86ca70df2e806b8d1 (diff)
Update upstream source from tag 'upstream/18.6.0'
Update to upstream version '18.6.0' with Debian dir d980267f8cdfa321f5dd5cdc741ee3f83f3d6c88
-rw-r--r--.coveragerc11
-rw-r--r--MANIFEST.in3
-rw-r--r--PKG-INFO34
-rw-r--r--README.rst20
-rw-r--r--docs/api.rst160
-rw-r--r--docs/conf.py9
-rw-r--r--docs/examples/basic_post.py4
-rw-r--r--docs/examples/download_file.py2
-rw-r--r--docs/examples/iresource.py16
-rw-r--r--docs/examples/query_params.py30
-rw-r--r--docs/examples/response_history.py3
-rw-r--r--docs/examples/testing_seq.py56
-rw-r--r--docs/examples/using_cookies.py2
-rw-r--r--docs/howto.rst39
-rw-r--r--docs/index.rst33
-rw-r--r--docs/testing.rst57
-rw-r--r--requirements-dev.txt5
-rw-r--r--setup.cfg10
-rw-r--r--setup.py79
-rw-r--r--src/treq.egg-info/PKG-INFO (renamed from treq.egg-info/PKG-INFO)34
-rw-r--r--src/treq.egg-info/SOURCES.txt62
-rw-r--r--src/treq.egg-info/dependency_links.txt (renamed from treq.egg-info/dependency_links.txt)0
-rw-r--r--src/treq.egg-info/requires.txt12
-rw-r--r--src/treq.egg-info/top_level.txt (renamed from treq.egg-info/top_level.txt)0
-rw-r--r--src/treq/__init__.py (renamed from treq/__init__.py)6
-rw-r--r--src/treq/_utils.py (renamed from treq/_utils.py)0
-rw-r--r--src/treq/_version.py11
-rw-r--r--src/treq/api.py (renamed from treq/api.py)10
-rw-r--r--src/treq/auth.py (renamed from treq/auth.py)0
-rw-r--r--src/treq/client.py (renamed from treq/client.py)76
-rw-r--r--src/treq/content.py (renamed from treq/content.py)33
-rw-r--r--src/treq/multipart.py (renamed from treq/multipart.py)64
-rw-r--r--src/treq/response.py117
-rw-r--r--src/treq/test/__init__.py (renamed from treq/test/__init__.py)0
-rw-r--r--src/treq/test/local_httpbin/__init__.py0
-rw-r--r--src/treq/test/local_httpbin/child.py282
-rw-r--r--src/treq/test/local_httpbin/parent.py171
-rw-r--r--src/treq/test/local_httpbin/shared.py39
-rw-r--r--src/treq/test/local_httpbin/test/__init__.py0
-rw-r--r--src/treq/test/local_httpbin/test/test_child.py455
-rw-r--r--src/treq/test/local_httpbin/test/test_parent.py513
-rw-r--r--src/treq/test/local_httpbin/test/test_shared.py41
-rw-r--r--src/treq/test/test_api.py (renamed from treq/test/test_api.py)3
-rw-r--r--src/treq/test/test_auth.py (renamed from treq/test/test_auth.py)2
-rw-r--r--src/treq/test/test_client.py (renamed from treq/test/test_client.py)94
-rw-r--r--src/treq/test/test_content.py (renamed from treq/test/test_content.py)63
-rw-r--r--src/treq/test/test_multipart.py (renamed from treq/test/test_multipart.py)47
-rw-r--r--src/treq/test/test_response.py139
-rw-r--r--src/treq/test/test_testing.py (renamed from treq/test/test_testing.py)27
-rw-r--r--src/treq/test/test_treq_integration.py (renamed from treq/test/test_treq_integration.py)61
-rw-r--r--src/treq/test/test_utils.py (renamed from treq/test/test_utils.py)2
-rw-r--r--src/treq/test/util.py31
-rw-r--r--src/treq/testing.py (renamed from treq/testing.py)257
-rw-r--r--tox.ini38
-rwxr-xr-xtox2travis.py80
-rw-r--r--treq.egg-info/SOURCES.txt52
-rw-r--r--treq.egg-info/pbr.json1
-rw-r--r--treq.egg-info/requires.txt5
-rw-r--r--treq/_version1
-rw-r--r--treq/response.py48
-rw-r--r--treq/test/test_response.py64
-rw-r--r--treq/test/util.py47
62 files changed, 2862 insertions, 699 deletions
diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 0000000..bffc91e
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,11 @@
+[run]
+source =
+ treq
+branch = True
+
+
+[paths]
+source =
+ src/
+ .tox/*/lib/python*/site-packages/
+ .tox/pypy*/site-packages/
diff --git a/MANIFEST.in b/MANIFEST.in
index 7feb963..a56d8e8 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,6 +1,7 @@
-include treq/_version README.rst LICENSE tox.ini tox2travis.py requirements-dev.txt
+include README.rst LICENSE tox.ini tox2travis.py .coveragerc
recursive-include docs *
prune docs/_build
exclude .travis.yml
+exclude .readthedocs.yml
global-exclude .DS_Store *.pyc
diff --git a/PKG-INFO b/PKG-INFO
index 3ebd400..ad9ef9f 100644
--- a/PKG-INFO
+++ b/PKG-INFO
@@ -1,10 +1,12 @@
-Metadata-Version: 1.1
+Metadata-Version: 2.1
Name: treq
-Version: 15.1.0
+Version: 18.6.0
Summary: A requests-like API built on top of twisted.web's Agent
-Home-page: http://github.com/twisted/treq
-Author: Amber Brown
-Author-email: hawkowl@twistedmatrix.com
+Home-page: https://github.com/twisted/treq
+Author: David Reid
+Author-email: dreid@dreid.org
+Maintainer: Amber Brown
+Maintainer-email: hawkowl@twistedmatrix.com
License: MIT/X
Description: treq
====
@@ -51,13 +53,7 @@ Description: treq
::
- pip install -r requirements-dev.txt
-
- Optionally install PyOpenSSL:
-
- ::
-
- pip install PyOpenSSL
+ pip install treq[dev]
Run Tests (unit & integration):
@@ -72,19 +68,17 @@ Description: treq
pep8 treq
pyflakes treq
- Build docs:
-
- ::
+ Build docs::
- cd docs; make html
+ tox -e docs
- .. |build| image:: https://secure.travis-ci.org/twisted/treq.svg?branch=master
- .. _build: http://travis-ci.org/twisted/treq
+ .. |build| image:: https://api.travis-ci.org/twisted/treq.svg?branch=master
+ .. _build: https://travis-ci.org/twisted/treq
.. |coverage| image:: https://codecov.io/github/twisted/treq/coverage.svg?branch=master
.. _coverage: https://codecov.io/github/twisted/treq
- .. |pypi| image:: http://img.shields.io/pypi/v/treq.svg
+ .. |pypi| image:: https://img.shields.io/pypi/v/treq.svg
.. _pypi: https://pypi.python.org/pypi/treq
Platform: UNKNOWN
@@ -95,8 +89,8 @@ Classifier: Operating System :: OS Independent
Classifier: Framework :: Twisted
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3.3
Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
+Provides-Extra: dev
diff --git a/README.rst b/README.rst
index 166fbc3..34b4afa 100644
--- a/README.rst
+++ b/README.rst
@@ -43,13 +43,7 @@ Install dependencies:
::
- pip install -r requirements-dev.txt
-
-Optionally install PyOpenSSL:
-
-::
-
- pip install PyOpenSSL
+ pip install treq[dev]
Run Tests (unit & integration):
@@ -64,17 +58,15 @@ Lint:
pep8 treq
pyflakes treq
-Build docs:
-
-::
+Build docs::
- cd docs; make html
+ tox -e docs
-.. |build| image:: https://secure.travis-ci.org/twisted/treq.svg?branch=master
-.. _build: http://travis-ci.org/twisted/treq
+.. |build| image:: https://api.travis-ci.org/twisted/treq.svg?branch=master
+.. _build: https://travis-ci.org/twisted/treq
.. |coverage| image:: https://codecov.io/github/twisted/treq/coverage.svg?branch=master
.. _coverage: https://codecov.io/github/twisted/treq
-.. |pypi| image:: http://img.shields.io/pypi/v/treq.svg
+.. |pypi| image:: https://img.shields.io/pypi/v/treq.svg
.. _pypi: https://pypi.python.org/pypi/treq
diff --git a/docs/api.rst b/docs/api.rst
index 31f8f94..d2677e5 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -1,5 +1,10 @@
+API Reference
+=============
+
+This page lists all of the interfaces exposed by the `treq` package.
+
Making Requests
-===============
+---------------
.. module:: treq
@@ -12,73 +17,146 @@ Making Requests
.. autofunction:: delete
Accessing Content
-=================
+-----------------
.. autofunction:: collect
.. autofunction:: content
.. autofunction:: text_content
.. autofunction:: json_content
-Responses
-=========
+HTTPClient Objects
+------------------
+
+.. module:: treq.client
+
+The :class:`treq.client.HTTPClient` class provides the same interface as the :mod:`treq` module itself.
+
+.. autoclass:: HTTPClient
+
+ .. automethod:: request
+ .. automethod:: get
+ .. automethod:: head
+ .. automethod:: post
+ .. automethod:: put
+ .. automethod:: patch
+ .. automethod:: delete
+
+Augmented Response Objects
+--------------------------
+
+:func:`treq.request`, :func:`treq.get`, etc. return an object which provides :class:`twisted.web.iweb.IResponse`, plus a few additional convenience methods:
.. module:: treq.response
-.. class:: Response
+.. class:: _Response
- .. method:: collect(collector)
+ .. automethod:: collect
+ .. automethod:: content
+ .. automethod:: json
+ .. automethod:: text
+ .. automethod:: history
+ .. automethod:: cookies
- Incrementally collect the body of the response.
+ Inherited from :class:`twisted.web.iweb.IResponse`:
- :param collector: A single argument callable that will be called
- with chunks of body data as it is received.
+ :ivar version: See :attr:`IResponse.version <twisted.web.iweb.IResponse.version>`
+ :ivar code: See :attr:`IResponse.code <twisted.web.iweb.IResponse.code>`
+ :ivar phrase: See :attr:`IResponse.phrase <twisted.web.iweb.IResponse.pharse>`
+ :ivar headers: See :attr:`IResponse.headers <twisted.web.iweb.IResponse.headers>`
+ :ivar length: See :attr:`IResponse.length <twisted.web.iweb.IResponse.length>`
+ :ivar request: See :attr:`IResponse.request <twisted.web.iweb.IResponse.request>`
+ :ivar previousResponse: See :attr:`IResponse.previousResponse <twisted.web.iweb.IResponse.previousResponse>`
- :returns: A `Deferred` that fires when the entire body has been
- received.
+ .. method:: deliverBody(protocol)
- .. method:: content()
+ See :meth:`IResponse.deliverBody() <twisted.web.iweb.IResponse.deliverBody>`
- Read the entire body all at once.
+ .. method:: setPreviousResponse(response)
- :returns: A `Deferred` that fires with a `bytes` object when the entire
- body has been received.
+ See :meth:`IResponse.setPreviousResponse() <twisted.web.iweb.IResponse.setPreviousResponse>`
- .. method:: text(encoding='ISO-8859-1')
- Read the entire body all at once as text.
- :param encoding: An encoding for the body, if none is given the
- encoding will be guessed, defaulting to this argument.
+Test Helpers
+------------
- :returns: A `Deferred` that fires with a `unicode` object when the
- entire body has been received.
+The :mod:`treq.testing` module contains tools for in-memory testing of HTTP clients and servers.
- .. method:: json()
+StubTreq Objects
+~~~~~~~~~~~~~~~~
- Read the entire body all at once and decode it as JSON.
+.. class:: treq.testing.StubTreq(resource)
- :returns: A `Deferred` that fires with the result of `json.loads` on
- the body after it has been received.
+ :class:`StubTreq` implements the same interface as the :mod:`treq` module
+ or the :class:`~treq.client.HTTPClient` class, with the limitation that it
+ does not support the ``files`` argument.
- .. method:: history()
+ .. method:: flush()
- Get a list of all responses that (such as intermediate redirects),
- that ultimately ended in the current response.
+ Flush all data between pending client/server pairs.
- :returns: A `list` of :class:`treq.response.Response` objects.
+ This is only necessary if a :obj:`Resource` under test returns
+ :obj:`NOT_DONE_YET` from its ``render`` method, making a response
+ asynchronous. In that case, after each write from the server,
+ :meth:`flush()` must be called so the client can see it.
- .. method:: cookies()
+ As the methods on :class:`treq.client.HTTPClient`:
- :returns: A `CookieJar`.
+ .. method:: request
- Inherited from twisted.web.iweb.IResponse.
+ See :func:`treq.request()`.
- .. attribute:: version
- .. attribute:: code
- .. attribute:: phrase
- .. attribute:: headers
- .. attribute:: length
- .. attribute:: request
- .. attribute:: previousResponse
+ .. method:: get
- .. method:: deliverBody(protocol)
- .. method:: setPreviousResponse(response)
+ See :func:`treq.get()`.
+
+ .. method:: head
+
+ See :func:`treq.head()`.
+
+ .. method:: post
+
+ See :func:`treq.post()`.
+
+ .. method:: put
+
+ See :func:`treq.put()`.
+
+ .. method:: patch
+
+ See :func:`treq.patch()`.
+
+ .. method:: delete
+
+ See :func:`treq.delete()`.
+
+RequestTraversalAgent Objects
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: treq.testing.RequestTraversalAgent
+ :members:
+
+RequestSequence Objects
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: treq.testing.RequestSequence
+ :members:
+
+StringStubbingResource Objects
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: treq.testing.StringStubbingResource
+ :members:
+
+HasHeaders Objects
+~~~~~~~~~~~~~~~~~~
+
+.. autoclass:: treq.testing.HasHeaders
+ :members:
+
+MultiPartProducer Objects
+-------------------------
+
+:class:`treq.multipart.MultiPartProducer` is used internally when making requests which involve files.
+
+.. automodule:: treq.multipart
+ :members:
diff --git a/docs/conf.py b/docs/conf.py
index c36efd2..38fe651 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -25,7 +25,7 @@ sys.path.insert(0, os.path.abspath('..'))
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['sphinx.ext.viewcode', 'sphinx.ext.autodoc']
+extensions = ['sphinx.ext.viewcode', 'sphinx.ext.autodoc', 'sphinx.ext.intersphinx']
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
@@ -48,7 +48,7 @@ copyright = u'2014, David Reid'
# built documents.
#
# The full version, including alpha/beta/rc tags.
-release = open('../treq/_version').readline().strip()
+from treq import __version__ as release
version = '.'.join(release.split('.')[:2])
# The language for content autogenerated by Sphinx. Refer to documentation
@@ -241,3 +241,8 @@ texinfo_documents = [
#texinfo_show_urls = 'footnote'
RTD_NEW_THEME = True
+
+intersphinx_mapping = {
+ 'python': ('https://docs.python.org/3.5', None),
+ 'twisted': ('https://twistedmatrix.com/documents/current/api/', None),
+}
diff --git a/docs/examples/basic_post.py b/docs/examples/basic_post.py
index dfc9477..520e04f 100644
--- a/docs/examples/basic_post.py
+++ b/docs/examples/basic_post.py
@@ -8,8 +8,8 @@ import treq
def main(reactor, *args):
d = treq.post('http://httpbin.org/post',
- json.dumps({"msg": "Hello!"}),
- headers={'Content-Type': ['application/json']})
+ json.dumps({"msg": "Hello!"}).encode('ascii'),
+ headers={b'Content-Type': [b'application/json']})
d.addCallback(print_response)
return d
diff --git a/docs/examples/download_file.py b/docs/examples/download_file.py
index 66d7c4c..4a11074 100644
--- a/docs/examples/download_file.py
+++ b/docs/examples/download_file.py
@@ -4,7 +4,7 @@ import treq
def download_file(reactor, url, destination_filename):
- destination = file(destination_filename, 'w')
+ destination = open(destination_filename, 'wb')
d = treq.get(url)
d.addCallback(treq.collect, destination.write)
d.addBoth(lambda _: destination.close())
diff --git a/docs/examples/iresource.py b/docs/examples/iresource.py
new file mode 100644
index 0000000..47a2026
--- /dev/null
+++ b/docs/examples/iresource.py
@@ -0,0 +1,16 @@
+import json
+
+from zope.interface import implementer
+from twisted.web.resource import IResource
+
+
+@implementer(IResource)
+class JsonResource(object):
+ isLeaf = True # NB: means getChildWithDefault will not be called
+
+ def __init__(self, data):
+ self.data = data
+
+ def render(self, request):
+ request.setHeader(b'Content-Type', b'application/json')
+ return json.dumps(self.data).encode('utf-8')
diff --git a/docs/examples/query_params.py b/docs/examples/query_params.py
index 2d56a39..65ab9cf 100644
--- a/docs/examples/query_params.py
+++ b/docs/examples/query_params.py
@@ -6,34 +6,34 @@ import treq
@inlineCallbacks
def main(reactor):
- print 'List of tuples'
+ print('List of tuples')
resp = yield treq.get('http://httpbin.org/get',
params=[('foo', 'bar'), ('baz', 'bax')])
- content = yield treq.content(resp)
- print content
+ content = yield resp.text()
+ print(content)
- print 'Single value dictionary'
+ print('Single value dictionary')
resp = yield treq.get('http://httpbin.org/get',
params={'foo': 'bar', 'baz': 'bax'})
- content = yield treq.content(resp)
- print content
+ content = yield resp.text()
+ print(content)
- print 'Multi value dictionary'
+ print('Multi value dictionary')
resp = yield treq.get('http://httpbin.org/get',
params={'foo': ['bar', 'baz', 'bax']})
- content = yield treq.content(resp)
- print content
+ content = yield resp.text()
+ print(content)
- print 'Mixed value dictionary'
+ print('Mixed value dictionary')
resp = yield treq.get('http://httpbin.org/get',
params={'foo': ['bar', 'baz'], 'bax': 'quux'})
- content = yield treq.content(resp)
- print content
+ content = yield resp.text()
+ print(content)
- print 'Preserved query parameters'
+ print('Preserved query parameters')
resp = yield treq.get('http://httpbin.org/get?foo=bar',
params={'baz': 'bax'})
- content = yield treq.content(resp)
- print content
+ content = yield resp.text()
+ print(content)
react(main, [])
diff --git a/docs/examples/response_history.py b/docs/examples/response_history.py
index 61a2c16..dfacc6c 100644
--- a/docs/examples/response_history.py
+++ b/docs/examples/response_history.py
@@ -8,7 +8,8 @@ def main(reactor, *args):
d = treq.get('http://httpbin.org/redirect/1')
def cb(response):
- print 'Response history:', response.history()
+ print('Response history:')
+ print(response.history())
return print_response(response)
d.addCallback(cb)
diff --git a/docs/examples/testing_seq.py b/docs/examples/testing_seq.py
new file mode 100644
index 0000000..8570010
--- /dev/null
+++ b/docs/examples/testing_seq.py
@@ -0,0 +1,56 @@
+from twisted.internet import defer
+from twisted.trial.unittest import SynchronousTestCase
+from twisted.web import http
+
+from treq.testing import StubTreq, HasHeaders
+from treq.testing import RequestSequence, StringStubbingResource
+
+
+@defer.inlineCallbacks
+def make_a_request(treq):
+ """
+ Make a request using treq.
+ """
+ response = yield treq.get('http://an.example/foo', params={'a': 'b'},
+ headers={b'Accept': b'application/json'})
+ if response.code == http.OK:
+ result = yield response.json()
+ else:
+ message = yield response.text()
+ raise Exception("Got an error from the server: {}".format(message))
+ defer.returnValue(result)
+
+
+class MakeARequestTests(SynchronousTestCase):
+ """
+ Test :func:`make_a_request()` using :mod:`treq.testing.RequestSequence`.
+ """
+
+ def test_200_ok(self):
+ """On a 200 response, return the response's JSON."""
+ req_seq = RequestSequence([
+ ((b'get', 'http://an.example/foo', {b'a': [b'b']},
+ HasHeaders({'Accept': ['application/json']}), b''),
+ (http.OK, {b'Content-Type': b'application/json'}, b'{"status": "ok"}'))
+ ])
+ treq = StubTreq(StringStubbingResource(req_seq))
+
+ with req_seq.consume(self.fail):
+ result = self.successResultOf(make_a_request(treq))
+
+ self.assertEqual({"status": "ok"}, result)
+
+ def test_418_teapot(self):
+ """On an unexpected response code, raise an exception"""
+ req_seq = RequestSequence([
+ ((b'get', 'http://an.example/foo', {b'a': [b'b']},
+ HasHeaders({'Accept': ['application/json']}), b''),
+ (418, {b'Content-Type': b'text/plain'}, b"I'm a teapot!"))
+ ])
+ treq = StubTreq(StringStubbingResource(req_seq))
+
+ with req_seq.consume(self.fail):
+ failure = self.failureResultOf(make_a_request(treq))
+
+ self.assertEqual(u"Got an error from the server: I'm a teapot!",
+ failure.getErrorMessage())
diff --git a/docs/examples/using_cookies.py b/docs/examples/using_cookies.py
index a963716..2cb2992 100644
--- a/docs/examples/using_cookies.py
+++ b/docs/examples/using_cookies.py
@@ -10,7 +10,7 @@ def main(reactor, *args):
def _get_jar(resp):
jar = resp.cookies()
- print 'The server set our hello cookie to: {0}'.format(jar['hello'])
+ print('The server set our hello cookie to: {}'.format(jar['hello']))
return treq.get('http://httpbin.org/cookies', cookies=jar)
diff --git a/docs/howto.rst b/docs/howto.rst
index 3b59722..f19ce27 100644
--- a/docs/howto.rst
+++ b/docs/howto.rst
@@ -1,16 +1,18 @@
+Use Cases
+=========
+
Handling Streaming Responses
----------------------------
-In addition to `receiving responses <http://twistedmatrix.com/documents/current/web/howto/client.html#auto4>`_
-with ``IResponse.deliverBody``.
-
-treq provides a helper function :py:func:`treq.collect` which takes a
-``response``, and a single argument function which will be called with all new
-data available from the response. Much like ``IProtocol.dataReceived``,
+In addition to `receiving responses <https://twistedmatrix.com/documents/current/web/howto/client.html#receiving-responses>`_
+with :meth:`IResponse.deliverBody`, treq provides a helper function
+:py:func:`treq.collect` which takes a
+``response`` and a single argument function which will be called with all new
+data available from the response. Much like :meth:`IProtocol.dataReceived`,
:py:func:`treq.collect` knows nothing about the framing of your data and will
simply call your collector function with any data that is currently available.
-Here is an example which simply a ``file`` object's write method to
+Here is an example which simply a file object's write method to
:py:func:`treq.collect` to save the response body to a file.
.. literalinclude:: examples/download_file.py
@@ -23,7 +25,7 @@ Query Parameters
----------------
:py:func:`treq.request` supports a ``params`` keyword argument which will
-be urlencoded and added to the ``url`` argument in addition to any query
+be URL-encoded and added to the ``url`` argument in addition to any query
parameters that may already exist.
The ``params`` argument may be either a ``dict`` or a ``list`` of
@@ -41,7 +43,7 @@ Full example: :download:`query_params.py <examples/query_params.py>`
Auth
----
-HTTP Basic authentication as specified in `RFC 2617`_ is easily supported by
+HTTP Basic authentication as specified in :rfc:`2617` is easily supported by
passing an ``auth`` keyword argument to any of the request functions.
The ``auth`` argument should be a tuple of the form ``('username', 'password')``.
@@ -52,8 +54,6 @@ The ``auth`` argument should be a tuple of the form ``('username', 'password')``
Full example: :download:`basic_auth.py <examples/basic_auth.py>`
-.. _RFC 2617: http://www.ietf.org/rfc/rfc2617.txt
-
Redirects
---------
@@ -77,7 +77,7 @@ any of the request methods.
Full example: :download:`disable_redirects.py <examples/disable_redirects.py>`
You can even access the complete history of treq response objects by calling
-the `history()` method on the the response.
+the :meth:`~treq.response._Response.history()` method on the response.
.. literalinclude:: examples/response_history.py
:linenos:
@@ -91,10 +91,10 @@ Cookies
Cookies can be set by passing a ``dict`` or ``cookielib.CookieJar`` instance
via the ``cookies`` keyword argument. Later cookies set by the server can be
-retrieved using the :py:func:`treq.cookies` function.
+retrieved using the :py:meth:`~treq.response._Response.cookies()` method.
-The the object returned by :py:func:`treq.cookies` supports the same key/value
-access as `requests cookies <http://requests.readthedocs.org/en/latest/user/quickstart/#cookies>`_
+The object returned by :py:meth:`~treq.response._Response.cookies()` supports the same key/value
+access as `requests cookies <http://requests.readthedocs.org/en/latest/user/quickstart/#cookies>`_.
.. literalinclude:: examples/using_cookies.py
:linenos:
@@ -115,3 +115,12 @@ A custom agent can be passed to the various treq request methods using the
custom_agent = Agent(reactor, connectTimeout=42)
treq.get(url, agent=custom_agent)
+
+Additionally a custom client can be instantiated to use a custom agent
+using the ``agent`` keyword argument:
+
+.. code-block:: python
+
+ custom_agent = Agent(reactor, connectTimeout=42)
+ client = treq.client.HTTPClient(agent=custom_agent)
+ client.get(url)
diff --git a/docs/index.rst b/docs/index.rst
index fce55e5..096858d 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -1,16 +1,15 @@
treq: High-level Twisted HTTP Client API
========================================
-treq depends on Twisted 12.3.0 or later and optionally pyOpenSSL.
-Python 3 support requires at least Twisted 15.5.
+`treq <https://pypi.python.org/pypi/treq>`_ depends on a recent Twisted and functions on Python 2.7 and Python 3.3+ (including PyPy).
Why?
----
`requests`_ by `Kenneth Reitz`_ is a wonderful library.
I want the same ease of use when writing Twisted applications.
-treq is not of course a perfect clone of `requests`.
-I have tried to stay true to the do-what-I-mean spirit of the `requests` API and also kept the API familiar to users of `Twisted`_ and ``twisted.web.client.Agent`` on which treq is based.
+treq is not of course a perfect clone of `requests`_.
+I have tried to stay true to the do-what-I-mean spirit of the `requests`_ API and also kept the API familiar to users of `Twisted`_ and :class:`twisted.web.client.Agent` on which treq is based.
.. _requests: http://python-requests.org/
.. _Kenneth Reitz: https://www.gittip.com/kennethreitz/
@@ -28,7 +27,7 @@ GET
.. literalinclude:: examples/basic_get.py
:linenos:
- :lines: 7-11
+ :lines: 7-10
Full example: :download:`basic_get.py <examples/basic_get.py>`
@@ -45,23 +44,23 @@ Full example: :download:`basic_post.py <examples/basic_post.py>`
Why not 100% requests-alike?
----------------------------
-Initially when I started off working on treq I thought the API should look exactly like `requests`_ except anything that would involve the network would return a ``Deferred``.
+Initially when I started off working on treq I thought the API should look exactly like `requests`_ except anything that would involve the network would return a :class:`~twisted.internet.defer.Deferred`.
Over time while attempting to mimic the `requests`_ API it became clear that not enough code could be shared between `requests`_ and treq for it to be worth the effort to translate many of the usage patterns from `requests`_.
With the current version of treq I have tried to keep the API simple, yet remain familiar to users of Twisted and its lower-level HTTP libraries.
-Feature Parity w/ Requests
---------------------------
+Feature Parity with Requests
+----------------------------
-Even though mimicing the `requests`_ API is not a goal, supporting most of it's features is.
+Even though mimicking the `requests`_ API is not a goal, supporting most of its features is.
Here is a list of `requests`_ features and their status in treq.
+----------------------------------+----------+----------+
| | requests | treq |
+----------------------------------+----------+----------+
-| International Domains and URLs | yes | no |
+| International Domains and URLs | yes | yes |
+----------------------------------+----------+----------+
| Keep-Alive & Connection Pooling | yes | yes |
+----------------------------------+----------+----------+
@@ -92,21 +91,15 @@ Here is a list of `requests`_ features and their status in treq.
| Python 3.x | yes | yes |
+----------------------------------+----------+----------+
-Howto
------
+Table of Contents
+-----------------
.. toctree::
:maxdepth: 3
howto
-
-API Documentation
------------------
-
-.. toctree::
- :maxdepth: 2
-
- api
+ testing
+ api
Indices and tables
==================
diff --git a/docs/testing.rst b/docs/testing.rst
new file mode 100644
index 0000000..22a136e
--- /dev/null
+++ b/docs/testing.rst
@@ -0,0 +1,57 @@
+Testing Helpers
+===============
+
+The :mod:`treq.testing` module provides some tools for testing both HTTP clients which use the treq API and implementations of the `Twisted Web resource model <https://twistedmatrix.com/documents/current/api/twisted.web.resource.IResource.html>`_.
+
+Writing tests for HTTP clients
+------------------------------
+
+The :class:`~treq.testing.StubTreq` class implements the :mod:`treq` module interface (:func:`treq.get()`, :func:`treq.post()`, etc.) but runs all I/O via a :class:`~twisted.test.proto_helpers.MemoryReactor`.
+It wraps a :class:`twisted.web.resource.IResource` provider which handles each request.
+
+You can wrap a pre-existing `IResource` provider, or write your own.
+For example, the :class:`twisted.web.resource.ErrorPage` resource can produce an arbitrary HTTP status code.
+:class:`twisted.web.static.File` can serve files or directories.
+And you can easily achieve custom responses by writing trivial resources yourself:
+
+.. literalinclude:: examples/iresource.py
+ :linenos:
+ :pyobject: JsonResource
+
+However, those resources don't assert anything about the request.
+The :class:`~treq.testing.RequestSequence` and :class:`~treq.testing.StringStubbingResource` classes make it easy to construct a resource which encodes the expected request and response pairs.
+Do note that most parameters to these functions must be bytes—it's safest to use the ``b''`` string syntax, which works on both Python 2 and 3.
+
+For example:
+
+.. literalinclude:: examples/testing_seq.py
+ :linenos:
+
+This may be run with ``trial testing_seq.py``.
+Download: :download:`testing_seq.py <examples/testing_seq.py>`.
+
+Loosely matching the request
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you don't care about certain parts of the request, you can pass :data:`mock.ANY`, which compares equal to anything.
+This sequence matches a single GET request with any parameters or headers:
+
+.. code-block:: python
+
+ RequestSequence([
+ ((b'get', mock.ANY, mock.ANY, b''), (200, {}, b'ok'))
+ ])
+
+
+If you care about headers, use :class:`~treq.testing.HasHeaders` to make assertions about the headers present in the request.
+It compares equal to a superset of the headers specified, which helps make your test robust to changes in treq or Agent.
+Right now treq adds the ``Accept-Encoding: gzip`` header, but as support for additional compression methods is added, this may change.
+
+Writing tests for Twisted Web resources
+---------------------------------------
+
+Since :class:`~treq.testing.StubTreq` wraps any resource, you can use it to test your server-side code as well.
+This is superior to calling your resource's methods directly or passing mock objects, since it uses a real :class:`~twisted.web.client.Agent` to generate the request and a real :class:`~twisted.web.server.Site` to process the response.
+Thus, the ``request`` object your code interacts with is a *real* :class:`twisted.web.server.Request` and behaves the same as it would in production.
+
+Note that if your resource returns :data:`~twisted.web.server.NOT_DONE_YET` you must keep a reference to the :class:`~treq.testing.RequestTraversalAgent` and call its :meth:`~treq.testing.RequestTraversalAgent.flush()` method to spin the memory reactor once the server writes additional data before the client will receive it.
diff --git a/requirements-dev.txt b/requirements-dev.txt
deleted file mode 100644
index db7a829..0000000
--- a/requirements-dev.txt
+++ /dev/null
@@ -1,5 +0,0 @@
--e .
-pyflakes
-pep8
-sphinx
-mock==1.0.1 # Can be removed once Python 2.6 is dropped.
diff --git a/setup.cfg b/setup.cfg
index ba6350b..adf5ed7 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,17 +1,7 @@
[bdist_wheel]
universal = 1
-[metadata]
-requires-dist =
- pyOpenSSL >= 0.15.1; python_version > '3.0'
- pyOpenSSL >= 0.13; python_version < '3.0'
- requests >= 2.1.0
- service_identity >=14.0.0
- Twisted >= 15.5.0; python_version > '3.0'
- Twisted >= 14.0.2; python_version < '3.0'
-
[egg_info]
tag_build =
tag_date = 0
-tag_svn_revision = 0
diff --git a/setup.py b/setup.py
index ea9a53d..139eeec 100644
--- a/setup.py
+++ b/setup.py
@@ -1,10 +1,4 @@
from setuptools import find_packages, setup
-import os.path
-import sys
-
-
-with open(os.path.join(os.path.dirname(__file__), "treq", "_version")) as ver:
- __version__ = ver.readline().strip()
classifiers = [
"Development Status :: 5 - Production/Stable",
@@ -14,44 +8,47 @@ classifiers = [
"Framework :: Twisted",
"Programming Language :: Python",
"Programming Language :: Python :: 2.7",
- "Programming Language :: Python :: 3.3",
"Programming Language :: Python :: 3.4",
"Programming Language :: Python :: 3.5",
"Programming Language :: Python :: Implementation :: CPython",
"Programming Language :: Python :: Implementation :: PyPy",
]
-with open('README.rst') as f:
- readme = f.read()
-
-PY3 = (sys.version_info[0] >= 3)
-
-install_requires = [
- "requests >= 2.1.0",
- "service_identity >= 14.0.0",
- "six"
-]
-
-if PY3:
- install_requires.append("Twisted >= 15.5.0")
- install_requires.append("pyOpenSSL >= 0.15.1")
-else:
- install_requires.append("Twisted >= 14.0.2")
- install_requires.append("pyOpenSSL >= 0.13")
-
-setup(
- name="treq",
- version=__version__,
- packages=find_packages(),
- install_requires=install_requires,
- package_data={"treq": ["_version"]},
- author="David Reid",
- author_email="dreid@dreid.org",
- maintainer="Amber Brown",
- maintainer_email="hawkowl@twistedmatrix.com",
- classifiers=classifiers,
- description="A requests-like API built on top of twisted.web's Agent",
- license="MIT/X",
- url="http://github.com/twisted/treq",
- long_description=readme
-)
+if __name__ == "__main__":
+
+ with open('README.rst') as f:
+ readme = f.read()
+
+ setup(
+ name="treq",
+ packages=find_packages('src'),
+ package_dir={"": "src"},
+ setup_requires=["incremental"],
+ use_incremental=True,
+ install_requires=[
+ "incremental",
+ "requests >= 2.1.0",
+ "six",
+ "Twisted[tls] >= 16.4.0",
+ "attrs",
+ ],
+ extras_require={
+ "dev": [
+ "mock",
+ "pep8",
+ "pyflakes",
+ "sphinx",
+ "httpbin==0.5.0",
+ ],
+ },
+ package_data={"treq": ["_version"]},
+ author="David Reid",
+ author_email="dreid@dreid.org",
+ maintainer="Amber Brown",
+ maintainer_email="hawkowl@twistedmatrix.com",
+ classifiers=classifiers,
+ description="A requests-like API built on top of twisted.web's Agent",
+ license="MIT/X",
+ url="https://github.com/twisted/treq",
+ long_description=readme
+ )
diff --git a/treq.egg-info/PKG-INFO b/src/treq.egg-info/PKG-INFO
index 3ebd400..ad9ef9f 100644
--- a/treq.egg-info/PKG-INFO
+++ b/src/treq.egg-info/PKG-INFO
@@ -1,10 +1,12 @@
-Metadata-Version: 1.1
+Metadata-Version: 2.1
Name: treq
-Version: 15.1.0
+Version: 18.6.0
Summary: A requests-like API built on top of twisted.web's Agent
-Home-page: http://github.com/twisted/treq
-Author: Amber Brown
-Author-email: hawkowl@twistedmatrix.com
+Home-page: https://github.com/twisted/treq
+Author: David Reid
+Author-email: dreid@dreid.org
+Maintainer: Amber Brown
+Maintainer-email: hawkowl@twistedmatrix.com
License: MIT/X
Description: treq
====
@@ -51,13 +53,7 @@ Description: treq
::
- pip install -r requirements-dev.txt
-
- Optionally install PyOpenSSL:
-
- ::
-
- pip install PyOpenSSL
+ pip install treq[dev]
Run Tests (unit & integration):
@@ -72,19 +68,17 @@ Description: treq
pep8 treq
pyflakes treq
- Build docs:
-
- ::
+ Build docs::
- cd docs; make html
+ tox -e docs
- .. |build| image:: https://secure.travis-ci.org/twisted/treq.svg?branch=master
- .. _build: http://travis-ci.org/twisted/treq
+ .. |build| image:: https://api.travis-ci.org/twisted/treq.svg?branch=master
+ .. _build: https://travis-ci.org/twisted/treq
.. |coverage| image:: https://codecov.io/github/twisted/treq/coverage.svg?branch=master
.. _coverage: https://codecov.io/github/twisted/treq
- .. |pypi| image:: http://img.shields.io/pypi/v/treq.svg
+ .. |pypi| image:: https://img.shields.io/pypi/v/treq.svg
.. _pypi: https://pypi.python.org/pypi/treq
Platform: UNKNOWN
@@ -95,8 +89,8 @@ Classifier: Operating System :: OS Independent
Classifier: Framework :: Twisted
Classifier: Programming Language :: Python
Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3.3
Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
+Provides-Extra: dev
diff --git a/src/treq.egg-info/SOURCES.txt b/src/treq.egg-info/SOURCES.txt
new file mode 100644
index 0000000..cb21648
--- /dev/null
+++ b/src/treq.egg-info/SOURCES.txt
@@ -0,0 +1,62 @@
+.coveragerc
+LICENSE
+MANIFEST.in
+README.rst
+setup.cfg
+setup.py
+tox.ini
+tox2travis.py
+docs/Makefile
+docs/api.rst
+docs/conf.py
+docs/howto.rst
+docs/index.rst
+docs/make.bat
+docs/testing.rst
+docs/_static/.keepme
+docs/examples/_utils.py
+docs/examples/basic_auth.py
+docs/examples/basic_get.py
+docs/examples/basic_post.py
+docs/examples/disable_redirects.py
+docs/examples/download_file.py
+docs/examples/iresource.py
+docs/examples/query_params.py
+docs/examples/redirects.py
+docs/examples/response_history.py
+docs/examples/testing_seq.py
+docs/examples/using_cookies.py
+src/treq/__init__.py
+src/treq/_utils.py
+src/treq/_version.py
+src/treq/api.py
+src/treq/auth.py
+src/treq/client.py
+src/treq/content.py
+src/treq/multipart.py
+src/treq/response.py
+src/treq/testing.py
+src/treq.egg-info/PKG-INFO
+src/treq.egg-info/SOURCES.txt
+src/treq.egg-info/dependency_links.txt
+src/treq.egg-info/requires.txt
+src/treq.egg-info/top_level.txt
+src/treq/test/__init__.py
+src/treq/test/test_api.py
+src/treq/test/test_auth.py
+src/treq/test/test_client.py
+src/treq/test/test_content.py
+src/treq/test/test_multipart.py
+src/treq/test/test_response.py
+src/treq/test/test_testing.py
+src/treq/test/test_treq_integration.py
+src/treq/test/test_utils.py
+src/treq/test/util.py
+src/treq/test/local_httpbin/__init__.py
+src/treq/test/local_httpbin/child.py
+src/treq/test/local_httpbin/parent.py
+src/treq/test/local_httpbin/shared.py
+src/treq/test/local_httpbin/test/__init__.py
+src/treq/test/local_httpbin/test/test_child.py
+src/treq/test/local_httpbin/test/test_parent.py
+src/treq/test/local_httpbin/test/test_shared.py \ No newline at end of file
diff --git a/treq.egg-info/dependency_links.txt b/src/treq.egg-info/dependency_links.txt
index 8b13789..8b13789 100644
--- a/treq.egg-info/dependency_links.txt
+++ b/src/treq.egg-info/dependency_links.txt
diff --git a/src/treq.egg-info/requires.txt b/src/treq.egg-info/requires.txt
new file mode 100644
index 0000000..7dbf9f6
--- /dev/null
+++ b/src/treq.egg-info/requires.txt
@@ -0,0 +1,12 @@
+incremental
+requests>=2.1.0
+six
+Twisted[tls]>=16.4.0
+attrs
+
+[dev]
+mock
+pep8
+pyflakes
+sphinx
+httpbin==0.5.0
diff --git a/treq.egg-info/top_level.txt b/src/treq.egg-info/top_level.txt
index 3073f7b..3073f7b 100644
--- a/treq.egg-info/top_level.txt
+++ b/src/treq.egg-info/top_level.txt
diff --git a/treq/__init__.py b/src/treq/__init__.py
index b231f59..771fb88 100644
--- a/treq/__init__.py
+++ b/src/treq/__init__.py
@@ -1,11 +1,11 @@
from __future__ import absolute_import, division, print_function
-from pkg_resources import resource_string
+from ._version import __version__
from treq.api import head, get, post, put, patch, delete, request
from treq.content import collect, content, text_content, json_content
+__version__ = __version__.base()
+
__all__ = ['head', 'get', 'post', 'put', 'patch', 'delete', 'request',
'collect', 'content', 'text_content', 'json_content']
-
-__version__ = resource_string(__name__, "_version").strip()
diff --git a/treq/_utils.py b/src/treq/_utils.py
index 69719ff..69719ff 100644
--- a/treq/_utils.py
+++ b/src/treq/_utils.py
diff --git a/src/treq/_version.py b/src/treq/_version.py
new file mode 100644
index 0000000..f3dc424
--- /dev/null
+++ b/src/treq/_version.py
@@ -0,0 +1,11 @@
+"""
+Provides treq version information.
+"""
+
+# This file is auto-generated! Do not edit!
+# Use `python -m incremental.update treq` to change this file.
+
+from incremental import Version
+
+__version__ = Version('treq', 18, 6, 0)
+__all__ = ["__version__"]
diff --git a/treq/api.py b/src/treq/api.py
index c6a1162..915a9bf 100644
--- a/treq/api.py
+++ b/src/treq/api.py
@@ -81,6 +81,9 @@ def request(method, url, **kwargs):
:param data: Optional request body.
:type data: str, file-like, IBodyProducer, or None
+ :param json: Optional JSON-serializable content to pass in body.
+ :type json: dict, list/tuple, int, string/unicode, bool, or None
+
:param reactor: Optional twisted reactor.
:param bool persistent: Use persistent HTTP connections. Default: ``True``
@@ -97,6 +100,13 @@ def request(method, url, **kwargs):
received within this timeframe, a connection is aborted with
``CancelledError``.
+ :param bool browser_like_redirects: Use browser like redirects
+ (i.e. Ignore RFC2616 section 10.3 and follow redirects from
+ POST requests). Default: ``False``
+
+ :param bool unbuffered: Pass ``True`` to to disable response buffering. By
+ default treq buffers the entire response body in memory.
+
:rtype: Deferred that fires with an IResponse provider.
"""
diff --git a/treq/auth.py b/src/treq/auth.py
index d693ff3..d693ff3 100644
--- a/treq/auth.py
+++ b/src/treq/auth.py
diff --git a/treq/client.py b/src/treq/client.py
index 1bf78e5..61357a5 100644
--- a/treq/client.py
+++ b/src/treq/client.py
@@ -10,42 +10,41 @@ from twisted.internet.defer import Deferred
from twisted.python.components import proxyForInterface
from twisted.python.compat import _PY3, unicode
from twisted.python.filepath import FilePath
+from twisted.python.url import URL
from twisted.web.http import urlparse
-if _PY3:
- from urllib.parse import urlunparse, urlencode as _urlencode
-
- def urlencode(query, doseq):
- return _urlencode(query, doseq).encode('ascii')
-else:
- from urlparse import urlunparse
- from urllib import urlencode
-
from twisted.web.http_headers import Headers
from twisted.web.iweb import IBodyProducer, IResponse
from twisted.web.client import (
FileBodyProducer,
RedirectAgent,
+ BrowserLikeRedirectAgent,
ContentDecoderAgent,
GzipDecoder,
CookieAgent
)
from twisted.python.components import registerAdapter
+from json import dumps as json_dumps
from treq._utils import default_reactor
from treq.auth import add_auth
from treq import multipart
from treq.response import _Response
+from requests.cookies import cookiejar_from_dict, merge_cookies
if _PY3:
+ from urllib.parse import urlunparse, urlencode as _urlencode
+
+ def urlencode(query, doseq):
+ return _urlencode(query, doseq).encode('ascii')
from http.cookiejar import CookieJar
else:
from cookielib import CookieJar
-
-from requests.cookies import cookiejar_from_dict, merge_cookies
+ from urlparse import urlunparse
+ from urllib import urlencode
class _BodyBufferingProtocol(proxyForInterface(IProtocol)):
@@ -107,24 +106,45 @@ class HTTPClient(object):
self._data_to_body_producer = data_to_body_producer
def get(self, url, **kwargs):
+ """
+ See :func:`treq.get()`.
+ """
return self.request('GET', url, **kwargs)
def put(self, url, data=None, **kwargs):
+ """
+ See :func:`treq.put()`.
+ """
return self.request('PUT', url, data=data, **kwargs)
def patch(self, url, data=None, **kwargs):
+ """
+ See :func:`treq.patch()`.
+ """
return self.request('PATCH', url, data=data, **kwargs)
def post(self, url, data=None, **kwargs):
+ """
+ See :func:`treq.post()`.
+ """
return self.request('POST', url, data=data, **kwargs)
def head(self, url, **kwargs):
+ """
+ See :func:`treq.head()`.
+ """
return self.request('HEAD', url, **kwargs)
def delete(self, url, **kwargs):
+ """
+ See :func:`treq.delete()`.
+ """
return self.request('DELETE', url, **kwargs)
def request(self, method, url, **kwargs):
+ """
+ See :func:`treq.request()`.
+ """
method = method.encode('ascii').upper()
# Join parameters provided in the URL
@@ -134,7 +154,7 @@ class HTTPClient(object):
url = _combine_query_params(url, params)
if isinstance(url, unicode):
- url = url.encode('ascii')
+ url = URL.fromText(url).asURI().asText().encode('ascii')
# Convert headers dictionary to
# twisted raw headers format.
@@ -143,23 +163,9 @@ class HTTPClient(object):
if isinstance(headers, dict):
h = Headers({})
for k, v in headers.items():
-
- if isinstance(k, unicode):
- k = k.encode('ascii')
-
- if isinstance(v, bytes):
+ if isinstance(v, (bytes, unicode)):
h.addRawHeader(k, v)
- elif isinstance(v, unicode):
- h.addRawHeader(k, v.encode('ascii'))
elif isinstance(v, list):
- cleanHeaders = []
- for item in v:
- if isinstance(item, unicode):
- cleanHeaders.append(item.encode('ascii'))
- else:
- cleanHeaders.append(item)
- h.setRawHeaders(k, cleanHeaders)
- else:
h.setRawHeaders(k, v)
headers = h
@@ -171,6 +177,10 @@ class HTTPClient(object):
bodyProducer = None
data = kwargs.get('data')
files = kwargs.get('files')
+ # since json=None needs to be serialized as 'null', we need to
+ # explicitly check kwargs for this key
+ has_json = 'json' in kwargs
+
if files:
# If the files keyword is present we will issue a
# multipart/form-data request as it suits better for cases
@@ -195,6 +205,13 @@ class HTTPClient(object):
b'content-type', [b'application/x-www-form-urlencoded'])
data = urlencode(data, doseq=True)
bodyProducer = self._data_to_body_producer(data)
+ elif has_json:
+ # If data is sent as json, set Content-Type as 'application/json'
+ headers.setRawHeaders(
+ b'content-type', [b'application/json; charset=UTF-8'])
+ content = kwargs['json']
+ json = json_dumps(content, separators=(u',', u':')).encode('utf-8')
+ bodyProducer = self._data_to_body_producer(json)
cookies = kwargs.get('cookies', {})
@@ -205,7 +222,10 @@ class HTTPClient(object):
wrapped_agent = CookieAgent(self._agent, cookies)
if kwargs.get('allow_redirects', True):
- wrapped_agent = RedirectAgent(wrapped_agent)
+ if kwargs.get('browser_like_redirects', False):
+ wrapped_agent = BrowserLikeRedirectAgent(wrapped_agent)
+ else:
+ wrapped_agent = RedirectAgent(wrapped_agent)
wrapped_agent = ContentDecoderAgent(wrapped_agent,
[(b'gzip', GzipDecoder)])
diff --git a/treq/content.py b/src/treq/content.py
index 5531e3c..c6ac140 100644
--- a/treq/content.py
+++ b/src/treq/content.py
@@ -3,17 +3,15 @@ from __future__ import absolute_import, division, print_function
import cgi
import json
-from twisted.python.compat import _PY3
from twisted.internet.defer import Deferred, succeed
from twisted.internet.protocol import Protocol
from twisted.web.client import ResponseDone
from twisted.web.http import PotentialDataLoss
-from twisted.web.http_headers import Headers
def _encoding_from_headers(headers):
- content_types = headers.getRawHeaders('content-type')
+ content_types = headers.getRawHeaders(u'content-type')
if content_types is None:
return None
@@ -24,6 +22,9 @@ def _encoding_from_headers(headers):
if 'charset' in params:
return params.get('charset').strip("'\"")
+ if content_type == 'application/json':
+ return 'UTF-8'
+
class _BodyCollector(Protocol):
def __init__(self, finished, collector):
@@ -81,7 +82,7 @@ def content(response):
return d
-def json_content(response):
+def json_content(response, **kwargs):
"""
Read the contents of an HTTP response and attempt to decode it as JSON.
@@ -90,14 +91,14 @@ def json_content(response):
:param IResponse response: The HTTP Response to get the contents of.
+ :param kwargs: Any keyword arguments accepted by :py:func:`json.loads`
+
:rtype: Deferred that fires with the decoded JSON.
"""
- if _PY3:
- d = text_content(response)
- else:
- d = content(response)
+ # RFC7159 (8.1): Default JSON character encoding is UTF-8
+ d = text_content(response, encoding='utf-8')
- d.addCallback(json.loads)
+ d.addCallback(lambda text: json.loads(text, **kwargs))
return d
@@ -107,20 +108,14 @@ def text_content(response, encoding='ISO-8859-1'):
charset, which may be guessed from the ``Content-Type`` header.
:param IResponse response: The HTTP Response to get the contents of.
- :param str encoding: An valid charset, such as ``UTF-8`` or ``ISO-8859-1``.
+ :param str encoding: A charset, such as ``UTF-8`` or ``ISO-8859-1``,
+ used if the response does not specify an encoding.
- :rtype: Deferred that fires with a unicode.
+ :rtype: Deferred that fires with a unicode string.
"""
def _decode_content(c):
- if _PY3:
- headers = Headers({
- key.decode('ascii'): [y.decode('ascii') for y in val]
- for key, val in response.headers.getAllRawHeaders()})
- else:
- headers = response.headers
-
- e = _encoding_from_headers(headers)
+ e = _encoding_from_headers(response.headers)
if e is not None:
return c.decode(e)
diff --git a/treq/multipart.py b/src/treq/multipart.py
index f93fcfe..04699b3 100644
--- a/treq/multipart.py
+++ b/src/treq/multipart.py
@@ -22,39 +22,37 @@ CRLF = b"\r\n"
@implementer(IBodyProducer)
class MultiPartProducer(object):
"""
- L{MultiPartProducer} takes parameters for HTTP Request
- produces bytes in multipart/form-data format defined
-
- U{Multipart<http://tools.ietf.org/html/rfc2388>}
- and
- U{Mime format<http://tools.ietf.org/html/rfc2046>}
+ :class:`MultiPartProducer` takes parameters for a HTTP request and
+ produces bytes in multipart/form-data format defined in :rfc:`2388` and
+ :rfc:`2046`.
The encoded request is produced incrementally and the bytes are
written to a consumer.
- Fields should have form: [(parameter name, value), ...]
+ Fields should have form: ``[(parameter name, value), ...]``
Accepted values:
* Unicode strings (in this case parameter will be encoded with utf-8)
- * Tuples with (file name, content-type, L{IBodyProducer} objects)
+ * Tuples with (file name, content-type,
+ :class:`~twisted.web.iweb.IBodyProducer` objects)
- Since MultiPart producer can accept L{IBodyProducer} like objects
- and these objects sometimes cannot be read from in an event-driven manner
- (e.g. L{FileBodyProducer} is passed in)
- L{FileBodyProducer} uses a L{Cooperator} instance to schedule reads from
- the undelying producers. This process is also paused and resumed based
- on notifications from the L{IConsumer} provider being written to.
+ Since :class:`MultiPartProducer` can accept objects like
+ :class:`~twisted.web.iweb.IBodyProducer` which cannot be read from in an
+ event-driven manner it uses uses a
+ :class:`~twisted.internet.task.Cooperator` instance to schedule reads
+ from the underlying producers. Reading is also paused and resumed based on
+ notifications from the :class:`IConsumer` provider being written to.
- @ivar _fileds: Sorted parameters, where all strings are enforced to be
- unicode and file objects stacked on bottom (to produce a human readable
- form-data request)
+ :ivar _fields: Sorted parameters, where all strings are enforced to be
+ unicode and file objects stacked on bottom (to produce a human readable
+ form-data request)
- @ivar _cooperate: A method like L{Cooperator.cooperate} which is used to
+ :ivar _cooperate: A method like `Cooperator.cooperate` which is used to
schedule all reads.
- @ivar boundary: The generated boundary used in form-data encoding
- @type boundary: L{bytes}
+ :ivar boundary: The generated boundary used in form-data encoding
+ :type boundary: `bytes`
"""
def __init__(self, fields, boundary=None, cooperator=task):
@@ -72,10 +70,10 @@ class MultiPartProducer(object):
def startProducing(self, consumer):
"""
Start a cooperative task which will read bytes from the input file and
- write them to C{consumer}. Return a L{Deferred} which fires after all
+ write them to `consumer`. Return a `Deferred` which fires after all
bytes have been written.
- @param consumer: Any L{IConsumer} provider
+ :param consumer: Any `IConsumer` provider
"""
self._task = self._cooperate(self._writeLoop(consumer))
d = self._task.whenDone()
@@ -89,7 +87,7 @@ class MultiPartProducer(object):
def stopProducing(self):
"""
Permanently stop writing bytes from the file to the consumer by
- stopping the underlying L{CooperativeTask}.
+ stopping the underlying `CooperativeTask`.
"""
if self._currentProducer:
self._currentProducer.stopProducing()
@@ -98,7 +96,7 @@ class MultiPartProducer(object):
def pauseProducing(self):
"""
Temporarily suspend copying bytes from the input file to the consumer
- by pausing the L{CooperativeTask} which drives that activity.
+ by pausing the `CooperativeTask` which drives that activity.
"""
if self._currentProducer:
# Having a current producer means that we are in
@@ -113,8 +111,8 @@ class MultiPartProducer(object):
def resumeProducing(self):
"""
- Undo the effects of a previous C{pauseProducing} and resume copying
- bytes to the consumer by resuming the L{CooperativeTask} which drives
+ Undo the effects of a previous `pauseProducing` and resume copying
+ bytes to the consumer by resuming the `CooperativeTask` which drives
the write activity.
"""
if self._currentProducer:
@@ -125,9 +123,9 @@ class MultiPartProducer(object):
def _calculateLength(self):
"""
Determine how many bytes the overall form post would consume.
- The easiest way is to calculate is to generate of C{fObj}
+ The easiest way is to calculate is to generate of `fObj`
(assuming it is not modified from this point on).
- If the determination cannot be made, return C{UNKNOWN_LENGTH}.
+ If the determination cannot be made, return `UNKNOWN_LENGTH`.
"""
consumer = _LengthConsumer()
for i in list(self._writeLoop(consumer)):
@@ -234,7 +232,7 @@ def _enforce_unicode(value):
This function enforces the stings passed to be unicode, so we won't
need to guess what's the encoding of the binary strings passed in.
If someone needs to pass the binary string, use BytesIO and wrap it with
- L{FileBodyProducer}
+ `FileBodyProducer`.
"""
if isinstance(value, unicode):
return value
@@ -282,13 +280,13 @@ def _converted(fields):
class _LengthConsumer(object):
"""
- L{_LengthConsumer} is used to calculate the length of the multi-part
+ `_LengthConsumer` is used to calculate the length of the multi-part
request. The easiest way to do that is to consume all the fields,
but instead writing them to the string just accumulate the request
length.
- @ivar length: The length of the request. Can be UNKNOWN_LENGTH
- if consumer finds the field that has length that can not be calculated
+ :ivar length: The length of the request. Can be `UNKNOWN_LENGTH`
+ if consumer finds the field that has length that can not be calculated
"""
@@ -312,7 +310,7 @@ class _LengthConsumer(object):
class _Header(object):
"""
- L{_Header} This class is a tiny wrapper that produces
+ `_Header` This class is a tiny wrapper that produces
request headers. We can't use standard python header
class because it encodes unicode fields using =? bla bla ?=
encoding, which is correct, but no one in HTTP world expects
diff --git a/src/treq/response.py b/src/treq/response.py
new file mode 100644
index 0000000..3689c8a
--- /dev/null
+++ b/src/treq/response.py
@@ -0,0 +1,117 @@
+from __future__ import absolute_import, division, print_function
+
+from twisted.python.components import proxyForInterface
+from twisted.web.iweb import IResponse, UNKNOWN_LENGTH
+from twisted.python import reflect
+
+from requests.cookies import cookiejar_from_dict
+
+from treq.content import collect, content, json_content, text_content
+
+
+class _Response(proxyForInterface(IResponse)):
+ """
+ A wrapper for :class:`twisted.web.iweb.IResponse` which manages cookies and
+ adds a few convenience methods.
+ """
+
+ def __init__(self, original, cookiejar):
+ self.original = original
+ self._cookiejar = cookiejar
+
+ def __repr__(self):
+ """
+ Generate a representation of the response which includes the HTTP
+ status code, Content-Type header, and body size, if available.
+ """
+ if self.original.length == UNKNOWN_LENGTH:
+ size = 'unknown size'
+ else:
+ size = '{:,d} bytes'.format(self.original.length)
+ # Display non-ascii bits of the content-type header as backslash
+ # escapes.
+ content_type_bytes = b', '.join(
+ self.original.headers.getRawHeaders(b'content-type', ()))
+ content_type = repr(content_type_bytes).lstrip('b')[1:-1]
+ return "<{} {} '{:.40s}' {}>".format(
+ reflect.qual(self.__class__),
+ self.original.code,
+ content_type,
+ size,
+ )
+
+ def collect(self, collector):
+ """
+ Incrementally collect the body of the response, per
+ :func:`treq.collect()`.
+
+ :param collector: A single argument callable that will be called
+ with chunks of body data as it is received.
+
+ :returns: A `Deferred` that fires when the entire body has been
+ received.
+ """
+ return collect(self.original, collector)
+
+ def content(self):
+ """
+ Read the entire body all at once, per :func:`treq.content()`.
+
+ :returns: A `Deferred` that fires with a `bytes` object when the entire
+ body has been received.
+ """
+ return content(self.original)
+
+ def json(self, **kwargs):
+ """
+ Collect the response body as JSON per :func:`treq.json_content()`.
+
+ :param kwargs: Any keyword arguments accepted by :py:func:`json.loads`
+
+ :rtype: Deferred that fires with the decoded JSON when the entire body
+ has been read.
+ """
+ return json_content(self.original, **kwargs)
+
+ def text(self, encoding='ISO-8859-1'):
+ """
+ Read the entire body all at once as text, per
+ :func:`treq.text_content()`.
+
+ :rtype: A `Deferred` that fires with a unicode string when the entire
+ body has been received.
+ """
+ return text_content(self.original, encoding)
+
+ def history(self):
+ """
+ Get a list of all responses that (such as intermediate redirects),
+ that ultimately ended in the current response. The responses are
+ ordered chronologically.
+
+ :returns: A `list` of :class:`~treq.response._Response` objects
+ """
+ response = self
+ history = []
+
+ while response.previousResponse is not None:
+ history.append(_Response(response.previousResponse,
+ self._cookiejar))
+ response = response.previousResponse
+
+ history.reverse()
+ return history
+
+ def cookies(self):
+ """
+ Get a copy of this response's cookies.
+
+ :rtype: :class:`requests.cookies.RequestsCookieJar`
+ """
+ jar = cookiejar_from_dict({})
+
+ if self._cookiejar is not None:
+ for cookie in self._cookiejar:
+ jar.set_cookie(cookie)
+
+ return jar
diff --git a/treq/test/__init__.py b/src/treq/test/__init__.py
index e69de29..e69de29 100644
--- a/treq/test/__init__.py
+++ b/src/treq/test/__init__.py
diff --git a/src/treq/test/local_httpbin/__init__.py b/src/treq/test/local_httpbin/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/treq/test/local_httpbin/__init__.py
diff --git a/src/treq/test/local_httpbin/child.py b/src/treq/test/local_httpbin/child.py
new file mode 100644
index 0000000..5b972b4
--- /dev/null
+++ b/src/treq/test/local_httpbin/child.py
@@ -0,0 +1,282 @@
+"""
+A local ``httpbin`` server to run integration tests against.
+
+This ensures tests do not depend on `httpbin <https://httpbin.org/>`_.
+"""
+from __future__ import print_function
+import argparse
+import datetime
+import sys
+
+import httpbin
+
+import six
+
+from twisted.internet.defer import Deferred, inlineCallbacks, returnValue
+from twisted.internet.endpoints import TCP4ServerEndpoint, SSL4ServerEndpoint
+from twisted.internet.task import react
+from twisted.internet.ssl import (Certificate,
+ CertificateOptions)
+
+from OpenSSL.crypto import PKey, X509
+
+from twisted.python.threadpool import ThreadPool
+from twisted.web.server import Site
+from twisted.web.wsgi import WSGIResource
+
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric import rsa
+
+from cryptography.x509.oid import NameOID
+from cryptography.hazmat.primitives.serialization import Encoding
+
+from .shared import _HTTPBinDescription
+
+
+def _certificates_for_authority_and_server(service_identity, key_size=1024):
+ """
+ Create a self-signed CA certificate and server certificate signed
+ by the CA.
+
+ :param service_identity: The identity (hostname) of the server.
+ :type service_identity: :py:class:`unicode`
+
+ :param key_size: (optional) The size of CA's and server's private
+ RSA keys. Defaults to 1024 bits, which is the minimum allowed
+ by OpenSSL Contexts at the default security level as of 1.1.
+ :type key_size: :py:class:`int`
+
+ :return: a 3-tuple of ``(certificate_authority_certificate,
+ server_private_key, server_certificate)``.
+ :rtype: :py:class:`tuple` of (:py:class:`sslverify.Certificate`,
+ :py:class:`OpenSSL.crypto.PKey`,
+ :py:class:`OpenSSL.crypto.X509`)
+ """
+ common_name_for_ca = x509.Name(
+ [x509.NameAttribute(NameOID.COMMON_NAME, u'Testing Example CA')]
+ )
+ common_name_for_server = x509.Name(
+ [x509.NameAttribute(NameOID.COMMON_NAME, u'Testing Example Server')]
+ )
+ one_day = datetime.timedelta(1, 0, 0)
+ private_key_for_ca = rsa.generate_private_key(
+ public_exponent=65537,
+ key_size=key_size,
+ backend=default_backend()
+ )
+ public_key_for_ca = private_key_for_ca.public_key()
+ ca_certificate = (
+ x509.CertificateBuilder()
+ .subject_name(common_name_for_ca)
+ .issuer_name(common_name_for_ca)
+ .not_valid_before(datetime.datetime.today() - one_day)
+ .not_valid_after(datetime.datetime.today() + one_day)
+ .serial_number(x509.random_serial_number())
+ .public_key(public_key_for_ca)
+ .add_extension(
+ x509.BasicConstraints(ca=True, path_length=9), critical=True,
+ )
+ .sign(
+ private_key=private_key_for_ca, algorithm=hashes.SHA256(),
+ backend=default_backend()
+ )
+ )
+ private_key_for_server = rsa.generate_private_key(
+ public_exponent=65537,
+ key_size=key_size,
+ backend=default_backend()
+ )
+ public_key_for_server = private_key_for_server.public_key()
+ server_certificate = (
+ x509.CertificateBuilder()
+ .subject_name(common_name_for_server)
+ .issuer_name(common_name_for_ca)
+ .not_valid_before(datetime.datetime.today() - one_day)
+ .not_valid_after(datetime.datetime.today() + one_day)
+ .serial_number(x509.random_serial_number())
+ .public_key(public_key_for_server)
+ .add_extension(
+ x509.BasicConstraints(ca=False, path_length=None), critical=True,
+ )
+ .add_extension(
+ x509.SubjectAlternativeName(
+ [x509.DNSName(service_identity)]
+ ),
+ critical=True,
+ )
+ .sign(
+ private_key=private_key_for_ca, algorithm=hashes.SHA256(),
+ backend=default_backend()
+ )
+ )
+
+ ca_self_cert = Certificate.loadPEM(
+ ca_certificate.public_bytes(Encoding.PEM)
+ )
+
+ pkey = PKey.from_cryptography_key(private_key_for_server)
+ x509_server_certificate = X509.from_cryptography(server_certificate)
+
+ return ca_self_cert, pkey, x509_server_certificate
+
+
+def _make_httpbin_site(reactor, threadpool_factory=ThreadPool):
+ """
+ Return a :py:class:`Site` that hosts an ``httpbin`` WSGI
+ application.
+
+ :param reactor: The reactor.
+ :param threadpool_factory: (optional) A callable that creates a
+ :py:class:`ThreadPool`.
+
+ :return: A :py:class:`Site` that hosts ``httpbin``
+ """
+ wsgi_threads = threadpool_factory()
+ wsgi_threads.start()
+ reactor.addSystemEventTrigger("before", "shutdown", wsgi_threads.stop)
+
+ wsgi_resource = WSGIResource(reactor, wsgi_threads, httpbin.app)
+
+ return Site(wsgi_resource)
+
+
+@inlineCallbacks
+def _serve_tls(reactor, host, port, site):
+ """
+ Serve a site over TLS.
+
+ :param reactor: The reactor.
+ :param host: The host on which to listen.
+ :type host: :py:class:`str`
+
+ :param port: The host on which to listen.
+ :type port: :py:class:`int`
+ :type site: The :py:class:`Site` to serve.
+
+ :return: A :py:class:`Deferred` that fires with a
+ :py:class:`_HTTPBinDescription`
+ """
+ cert_host = host.decode('ascii') if six.PY2 else host
+
+ (
+ ca_cert, private_key, certificate,
+ ) = _certificates_for_authority_and_server(cert_host)
+
+ context_factory = CertificateOptions(privateKey=private_key,
+ certificate=certificate)
+
+ endpoint = SSL4ServerEndpoint(reactor,
+ port,
+ sslContextFactory=context_factory,
+ interface=host)
+
+ port = yield endpoint.listen(site)
+
+ description = _HTTPBinDescription(host=host,
+ port=port.getHost().port,
+ cacert=ca_cert.dumpPEM().decode('ascii'))
+
+ returnValue(description)
+
+
+@inlineCallbacks
+def _serve_tcp(reactor, host, port, site):
+ """
+ Serve a site over plain TCP.
+
+ :param reactor: The reactor.
+ :param host: The host on which to listen.
+ :type host: :py:class:`str`
+
+ :param port: The host on which to listen.
+ :type port: :py:class:`int`
+
+ :return: A :py:class:`Deferred` that fires with a
+ :py:class:`_HTTPBinDescription`
+ """
+ endpoint = TCP4ServerEndpoint(reactor, port, interface=host)
+
+ port = yield endpoint.listen(site)
+
+ description = _HTTPBinDescription(host=host, port=port.getHost().port)
+
+ returnValue(description)
+
+
+def _output_process_description(description, stdout=sys.stdout):
+ """
+ Write a process description to standard out.
+
+ :param description: The process description.
+ :type description: :py:class:`_HTTPBinDescription`
+
+ :param stdout: (optional) Standard out.
+ """
+ if six.PY2:
+ write = stdout.write
+ flush = stdout.flush
+ else:
+ write = stdout.buffer.write
+ flush = stdout.buffer.flush
+
+ write(description.to_json_bytes() + b'\n')
+ flush()
+
+
+def _forever_httpbin(reactor, argv,
+ _make_httpbin_site=_make_httpbin_site,
+ _serve_tcp=_serve_tcp,
+ _serve_tls=_serve_tls,
+ _output_process_description=_output_process_description):
+ """
+ Run ``httpbin`` forever.
+
+ :param reactor: The Twisted reactor.
+ :param argv: The arguments with which the script was ran.
+ :type argv: :py:class:`list` of :py:class:`str`
+
+ :return: a :py:class:`Deferred` that never fires.
+ """
+ parser = argparse.ArgumentParser(
+ description="""
+ Run httpbin forever. This writes a JSON object to
+ standard out. The host and port properties
+ contain the host and port on which httpbin
+ listens. When run with HTTPS, the cacert property
+ contains the PEM-encode CA certificate that
+ clients must trust.
+ """
+ )
+ parser.add_argument("--https",
+ help="Serve HTTPS",
+ action="store_const",
+ dest='serve',
+ const=_serve_tls,
+ default=_serve_tcp)
+ parser.add_argument("--host",
+ help="The host on which the server will listen.",
+ type=str,
+ default="localhost")
+ parser.add_argument("--port",
+ help="The on which the server will listen.",
+ type=int,
+ default=0)
+
+ arguments = parser.parse_args(argv)
+
+ site = _make_httpbin_site(reactor)
+
+ description_deferred = arguments.serve(reactor,
+ arguments.host,
+ arguments.port,
+ site)
+ description_deferred.addCallback(_output_process_description)
+ description_deferred.addCallback(lambda _: Deferred())
+
+ return description_deferred
+
+
+if __name__ == '__main__':
+ react(_forever_httpbin, (sys.argv[1:],))
diff --git a/src/treq/test/local_httpbin/parent.py b/src/treq/test/local_httpbin/parent.py
new file mode 100644
index 0000000..bb940a2
--- /dev/null
+++ b/src/treq/test/local_httpbin/parent.py
@@ -0,0 +1,171 @@
+"""
+Spawn and monitor an ``httpbin`` child process.
+"""
+import attr
+
+import signal
+import sys
+import os
+
+
+from twisted.protocols import basic, policies
+from twisted.internet import protocol, endpoints, error
+from twisted.internet.defer import Deferred, succeed
+
+from .shared import _HTTPBinDescription
+
+
+class _HTTPBinServerProcessProtocol(basic.LineOnlyReceiver):
+ """
+ Manage the lifecycle of an ``httpbin`` process.
+ """
+ delimiter = b'\n'
+
+ def __init__(self, all_data_received, terminated):
+ """
+ Manage the lifecycle of an ``httpbin`` process.
+
+ :param all_data_received: A Deferred that will be called back
+ with an :py:class:`_HTTPBinDescription` object
+ :type all_data_received: :py:class:`Deferred`
+
+ :param terminated: A Deferred that will be called back when
+ the process has ended.
+ :type terminated: :py:class:`Deferred`
+ """
+ self._all_data_received = all_data_received
+ self._received = False
+ self._terminated = terminated
+
+ def lineReceived(self, line):
+ if self._received:
+ raise RuntimeError("Unexpected line: {!r}".format(line))
+ description = _HTTPBinDescription.from_json_bytes(line)
+
+ self._received = True
+
+ # Remove readers and writers that leave the reactor in a dirty
+ # state after a test.
+ self.transport.closeStdin()
+ self.transport.closeStdout()
+ self.transport.closeStderr()
+
+ self._all_data_received.callback(description)
+
+ def connectionLost(self, reason):
+ if not self._received:
+ self._all_data_received.errback(reason)
+ self._terminated.errback(reason)
+
+
+@attr.s
+class _HTTPBinProcess(object):
+ """
+ Manage an ``httpbin`` server process.
+
+ :ivar _all_data_received: See
+ :py:attr:`_HTTPBinServerProcessProtocol.all_data_received`
+ :ivar _terminated: See
+ :py:attr:`_HTTPBinServerProcessProtocol.terminated`
+ """
+ _https = attr.ib()
+
+ _error_log_path = attr.ib(default='httpbin-server-error.log')
+
+ _all_data_received = attr.ib(init=False, default=attr.Factory(Deferred))
+ _terminated = attr.ib(init=False, default=attr.Factory(Deferred))
+
+ _process = attr.ib(init=False, default=None)
+ _process_description = attr.ib(init=False, default=None)
+
+ _open = staticmethod(open)
+
+ def _spawn_httpbin_process(self, reactor):
+ """
+ Spawn an ``httpbin`` process, returning a :py:class:`Deferred`
+ that fires with the process transport and result.
+ """
+ server = _HTTPBinServerProcessProtocol(
+ all_data_received=self._all_data_received,
+ terminated=self._terminated
+ )
+
+ argv = [
+ sys.executable,
+ '-m',
+ 'treq.test.local_httpbin.child',
+ ]
+
+ if self._https:
+ argv.append('--https')
+
+ with self._open(self._error_log_path, 'wb') as error_log:
+ endpoint = endpoints.ProcessEndpoint(
+ reactor,
+ sys.executable,
+ argv,
+ env=os.environ,
+ childFDs={
+ 1: 'r',
+ 2: error_log.fileno(),
+ },
+ )
+ # Processes are spawned synchronously.
+ spawned = endpoint.connect(
+ # ProtocolWrapper, WrappingFactory's protocol, has a
+ # disconnecting attribute. See
+ # https://twistedmatrix.com/trac/ticket/6606
+ policies.WrappingFactory(
+ protocol.Factory.forProtocol(lambda: server),
+ ),
+ )
+
+ def wait_for_protocol(connected_protocol):
+ process = connected_protocol.transport
+ return self._all_data_received.addCallback(
+ return_result_and_process, process,
+ )
+
+ def return_result_and_process(description, process):
+ return description, process
+
+ return spawned.addCallback(wait_for_protocol)
+
+ def server_description(self, reactor):
+ """
+ Return a :py:class:`Deferred` that fires with the the process'
+ :py:class:`_HTTPBinDescription`, spawning the process if
+ necessary.
+ """
+ if self._process is None:
+ ready = self._spawn_httpbin_process(reactor)
+
+ def store_and_schedule_termination(description_and_process):
+ description, process = description_and_process
+
+ self._process = process
+ self._process_description = description
+
+ reactor.addSystemEventTrigger("before", "shutdown", self.kill)
+
+ return self._process_description
+
+ return ready.addCallback(store_and_schedule_termination)
+ else:
+ return succeed(self._process_description)
+
+ def kill(self):
+ """
+ Kill the ``httpbin`` process.
+ """
+ if not self._process:
+ return
+
+ self._process.signalProcess("KILL")
+
+ def suppress_process_terminated(exit_failure):
+ exit_failure.trap(error.ProcessTerminated)
+ if exit_failure.value.signal != signal.SIGKILL:
+ return exit_failure
+
+ return self._terminated.addErrback(suppress_process_terminated)
diff --git a/src/treq/test/local_httpbin/shared.py b/src/treq/test/local_httpbin/shared.py
new file mode 100644
index 0000000..f057225
--- /dev/null
+++ b/src/treq/test/local_httpbin/shared.py
@@ -0,0 +1,39 @@
+"""
+Things shared between the ``httpbin`` child and parent processes
+"""
+import attr
+import json
+
+
+@attr.s
+class _HTTPBinDescription(object):
+ """
+ Describe an ``httpbin`` process.
+
+ :param host: The host on which the process listens.
+ :type host: :py:class:`str`
+
+ :param port: The port on which the process listens.
+ :type port: :py:class:`int`
+
+ :param cacert: (optional) The PEM-encoded certificate authority's
+ certificate. The calling process' treq must trust this when
+ running HTTPS tests.
+ :type cacert: :py:class:`bytes` or :py:class:`None`
+ """
+ host = attr.ib()
+ port = attr.ib()
+ cacert = attr.ib(default=None)
+
+ @classmethod
+ def from_json_bytes(cls, json_data):
+ """
+ Deserialize an instance from JSON bytes.
+ """
+ return cls(**json.loads(json_data.decode('ascii')))
+
+ def to_json_bytes(self):
+ """
+ Serialize an instance from JSON bytes.
+ """
+ return json.dumps(attr.asdict(self), sort_keys=True).encode('ascii')
diff --git a/src/treq/test/local_httpbin/test/__init__.py b/src/treq/test/local_httpbin/test/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/treq/test/local_httpbin/test/__init__.py
diff --git a/src/treq/test/local_httpbin/test/test_child.py b/src/treq/test/local_httpbin/test/test_child.py
new file mode 100644
index 0000000..0349bc5
--- /dev/null
+++ b/src/treq/test/local_httpbin/test/test_child.py
@@ -0,0 +1,455 @@
+"""
+Tests for :py:mod:`treq.test.local_httpbin.child`
+"""
+import attr
+
+from cryptography.hazmat.primitives.asymmetric import padding
+
+import functools
+
+import io
+
+from twisted.trial.unittest import SynchronousTestCase
+from twisted.test.proto_helpers import MemoryReactor
+
+from twisted.internet import defer
+
+from treq.test.util import skip_on_windows_because_of_199
+
+from twisted.web.server import Site
+from twisted.web.resource import Resource
+
+from service_identity.cryptography import verify_certificate_hostname
+
+import six
+
+from .. import child, shared
+
+
+skip = skip_on_windows_because_of_199()
+
+
+class CertificatesForAuthorityAndServerTests(SynchronousTestCase):
+ """
+ Tests for :py:func:`child._certificates_for_authority_and_server`
+ """
+
+ def setUp(self):
+ self.hostname = u".example.org"
+ (
+ self.ca_cert,
+ self.server_private_key,
+ self.server_x509_cert,
+ ) = child._certificates_for_authority_and_server(
+ self.hostname,
+ )
+
+ def test_pkey_x509_paired(self):
+ """
+ The returned private key corresponds to the X.509
+ certificate's public key.
+ """
+ server_private_key = self.server_private_key.to_cryptography_key()
+ server_x509_cert = self.server_x509_cert.to_cryptography()
+
+ plaintext = b'plaintext'
+ ciphertext = server_x509_cert.public_key().encrypt(
+ plaintext,
+ padding.PKCS1v15(),
+ )
+
+ self.assertEqual(
+ server_private_key.decrypt(
+ ciphertext,
+ padding.PKCS1v15(),
+ ),
+ plaintext,
+ )
+
+ def test_ca_signed_x509(self):
+ """
+ The returned X.509 certificate was signed by the returned
+ certificate authority's certificate.
+ """
+ ca_cert = self.ca_cert.original.to_cryptography()
+ server_x509_cert = self.server_x509_cert.to_cryptography()
+
+ # Raises an InvalidSignature exception on failure.
+ ca_cert.public_key().verify(
+ server_x509_cert.signature,
+ server_x509_cert.tbs_certificate_bytes,
+ padding.PKCS1v15(),
+ server_x509_cert.signature_hash_algorithm
+ )
+
+ def test_x509_matches_hostname(self):
+ """
+ The returned X.509 certificate is valid for the hostname.
+ """
+ verify_certificate_hostname(
+ self.server_x509_cert.to_cryptography(),
+ self.hostname,
+ )
+
+
+@attr.s
+class FakeThreadPoolState(object):
+ """
+ State for :py:class:`FakeThreadPool`.
+ """
+ init_call_count = attr.ib(default=0)
+ start_call_count = attr.ib(default=0)
+
+
+@attr.s
+class FakeThreadPool(object):
+ """
+ A fake :py:class:`twisted.python.threadpool.ThreadPool`
+ """
+ _state = attr.ib()
+
+ def init(self):
+ self._state.init_call_count += 1
+ return self
+
+ def start(self):
+ """
+ See :py:meth:`twisted.python.threadpool.ThreadPool.start`
+ """
+ self._state.start_call_count += 1
+
+ def stop(self):
+ """
+ See :py:meth:`twisted.python.threadpool.ThreadPool.stop`
+ """
+
+
+class MakeHTTPBinSiteTests(SynchronousTestCase):
+ """
+ Tests for :py:func:`_make_httpbin_site`.
+ """
+
+ def setUp(self):
+ self.fake_threadpool_state = FakeThreadPoolState()
+ self.fake_threadpool = FakeThreadPool(self.fake_threadpool_state)
+ self.reactor = MemoryReactor()
+
+ def test_threadpool_management(self):
+ """
+ A thread pool is created that will be shut down when the
+ reactor shuts down.
+ """
+ child._make_httpbin_site(
+ self.reactor,
+ threadpool_factory=self.fake_threadpool.init,
+ )
+
+ self.assertEqual(self.fake_threadpool_state.init_call_count, 1)
+ self.assertEqual(self.fake_threadpool_state.start_call_count, 1)
+
+ self.assertEqual(len(self.reactor.triggers['before']['shutdown']), 1)
+ [(stop, _, _)] = self.reactor.triggers['before']['shutdown']
+
+ self.assertEqual(stop, self.fake_threadpool.stop)
+
+
+class ServeTLSTests(SynchronousTestCase):
+ """
+ Tests for :py:func:`_serve_tls`
+ """
+
+ def setUp(self):
+ self.reactor = MemoryReactor()
+ self.site = Site(Resource())
+
+ def test_tls_listener_matches_description(self):
+ """
+ An SSL listener is established on the requested host and port,
+ and the host, port, and CA certificate are returned in its
+ description.
+ """
+ expected_host = 'host'
+ expected_port = 123
+
+ description_deferred = child._serve_tls(
+ self.reactor,
+ host=expected_host,
+ port=expected_port,
+ site=self.site,
+ )
+
+ self.assertEqual(len(self.reactor.sslServers), 1)
+
+ [
+ (actual_port, actual_site, _, _, actual_host)
+ ] = self.reactor.sslServers
+
+ self.assertEqual(actual_host, expected_host)
+ self.assertEqual(actual_port, expected_port)
+ self.assertIs(actual_site, self.site)
+
+ description = self.successResultOf(description_deferred)
+
+ self.assertEqual(description.host, expected_host)
+ self.assertEqual(description.port, expected_port)
+ self.assertTrue(description.cacert)
+
+
+class ServeTCPTests(SynchronousTestCase):
+ """
+ Tests for :py:func:`_serve_tcp`
+ """
+
+ def setUp(self):
+ self.reactor = MemoryReactor()
+ self.site = Site(Resource)
+
+ def test_tcp_listener_matches_description(self):
+ """
+ A TCP listeneris established on the request host and port, and
+ the host and port are returned in its description.
+ """
+ expected_host = 'host'
+ expected_port = 123
+
+ description_deferred = child._serve_tcp(
+ self.reactor,
+ host=expected_host,
+ port=expected_port,
+ site=self.site,
+ )
+
+ self.assertEqual(len(self.reactor.tcpServers), 1)
+
+ [
+ (actual_port, actual_site, _, actual_host)
+ ] = self.reactor.tcpServers
+
+ self.assertEqual(actual_host, expected_host)
+ self.assertEqual(actual_port, expected_port)
+ self.assertIs(actual_site, self.site)
+
+ description = self.successResultOf(description_deferred)
+
+ self.assertEqual(description.host, expected_host)
+ self.assertEqual(description.port, expected_port)
+ self.assertFalse(description.cacert)
+
+
+@attr.s
+class FlushableBytesIOState(object):
+ """
+ State for :py:class:`FlushableBytesIO`
+ """
+ bio = attr.ib(default=attr.Factory(io.BytesIO))
+ flush_count = attr.ib(default=0)
+
+
+@attr.s
+class FlushableBytesIO(object):
+ """
+ A :py:class:`io.BytesIO` wrapper that records flushes.
+ """
+ _state = attr.ib()
+
+ def write(self, data):
+ self._state.bio.write(data)
+
+ def flush(self):
+ self._state.flush_count += 1
+
+
+if not six.PY2:
+ @attr.s
+ class BufferedStandardOut(object):
+ """
+ A standard out that whose ``buffer`` is a
+ :py:class:`FlushableBytesIO` instance.
+ """
+ buffer = attr.ib()
+
+
+class OutputProcessDescriptionTests(SynchronousTestCase):
+ """
+ Tests for :py:func:`_output_process_description`
+ """
+
+ def setUp(self):
+ self.stdout_state = FlushableBytesIOState()
+ self.stdout = FlushableBytesIO(self.stdout_state)
+ if not six.PY2:
+ self.stdout = BufferedStandardOut(self.stdout)
+
+ def test_description_written(self):
+ """
+ An :py:class:`shared._HTTPBinDescription` is written to
+ standard out and the line flushed.
+ """
+ description = shared._HTTPBinDescription(host="host",
+ port=123,
+ cacert="cacert")
+
+ child._output_process_description(description, self.stdout)
+
+ written = self.stdout_state.bio.getvalue()
+
+ self.assertEqual(
+ written,
+ b'{"cacert": "cacert", "host": "host", "port": 123}' + b'\n',
+ )
+
+ self.assertEqual(self.stdout_state.flush_count, 1)
+
+
+class ForeverHTTPBinTests(SynchronousTestCase):
+ """
+ Tests for :py:func:`_forever_httpbin`
+ """
+
+ def setUp(self):
+ self.make_httpbin_site_returns = Site(Resource())
+
+ self.serve_tcp_calls = []
+ self.serve_tcp_returns = defer.Deferred()
+
+ self.serve_tls_calls = []
+ self.serve_tls_returns = defer.Deferred()
+
+ self.output_process_description_calls = []
+ self.output_process_description_returns = None
+
+ self.reactor = MemoryReactor()
+
+ self.forever_httpbin = functools.partial(
+ child._forever_httpbin,
+ _make_httpbin_site=self.make_httpbin_site,
+ _serve_tcp=self.serve_tcp,
+ _serve_tls=self.serve_tls,
+ _output_process_description=self.output_process_description,
+ )
+
+ def make_httpbin_site(self, reactor, *args, **kwargs):
+ """
+ A fake :py:func:`child._make_httpbin_site`.
+ """
+ return self.make_httpbin_site_returns
+
+ def serve_tcp(self, reactor, host, port, site):
+ """
+ A fake :py:func:`child._serve_tcp`.
+ """
+ self.serve_tcp_calls.append((reactor, host, port, site))
+ return self.serve_tcp_returns
+
+ def serve_tls(self, reactor, host, port, site):
+ """
+ A fake :py:func:`child._serve_tls`.
+ """
+ self.serve_tls_calls.append((reactor, host, port, site))
+ return self.serve_tls_returns
+
+ def output_process_description(self, description, *args, **kwargs):
+ """
+ A fake :py:func:`child._output_process_description`
+ """
+ self.output_process_description_calls.append(description)
+ return self.output_process_description_returns
+
+ def assertDescriptionAndDeferred(self,
+ description_deferred,
+ forever_deferred):
+ """
+ Assert that firing ``description_deferred`` outputs the
+ description but that ``forever_deferred`` never fires.
+ """
+ description_deferred.callback("description")
+
+ self.assertEqual(self.output_process_description_calls,
+ ["description"])
+
+ self.assertNoResult(forever_deferred)
+
+ def test_default_arguments(self):
+ """
+ The default command line arguments host ``httpbin`` on
+ ``localhost`` and a randomly-assigned port, returning a
+ :py:class:`Deferred` that never fires.
+ """
+ deferred = self.forever_httpbin(self.reactor, [])
+
+ self.assertEqual(
+ self.serve_tcp_calls,
+ [
+ (self.reactor, 'localhost', 0, self.make_httpbin_site_returns)
+ ]
+ )
+
+ self.assertDescriptionAndDeferred(
+ description_deferred=self.serve_tcp_returns,
+ forever_deferred=deferred,
+ )
+
+ def test_https(self):
+ """
+ The ``--https`` command line argument serves ``httpbin`` over
+ HTTPS, returning a :py:class:`Deferred` that never fires.
+ """
+ deferred = self.forever_httpbin(self.reactor, ['--https'])
+
+ self.assertEqual(
+ self.serve_tls_calls,
+ [
+ (self.reactor, 'localhost', 0, self.make_httpbin_site_returns)
+ ]
+ )
+
+ self.assertDescriptionAndDeferred(
+ description_deferred=self.serve_tls_returns,
+ forever_deferred=deferred,
+ )
+
+ def test_host(self):
+ """
+ The ``--host`` command line argument serves ``httpbin`` on
+ provided host, returning a :py:class:`Deferred` that never
+ fires.
+ """
+ deferred = self.forever_httpbin(self.reactor,
+ ['--host', 'example.org'])
+
+ self.assertEqual(
+ self.serve_tcp_calls,
+ [
+ (
+ self.reactor,
+ 'example.org',
+ 0,
+ self.make_httpbin_site_returns,
+ )
+ ]
+ )
+
+ self.assertDescriptionAndDeferred(
+ description_deferred=self.serve_tcp_returns,
+ forever_deferred=deferred,
+ )
+
+ def test_port(self):
+ """
+ The ``--port`` command line argument serves ``httpbin`` on
+ the provided port, returning a :py:class:`Deferred` that never
+ fires.
+ """
+ deferred = self.forever_httpbin(self.reactor, ['--port', '91'])
+
+ self.assertEqual(
+ self.serve_tcp_calls,
+ [
+ (self.reactor, 'localhost', 91, self.make_httpbin_site_returns)
+ ]
+ )
+
+ self.assertDescriptionAndDeferred(
+ description_deferred=self.serve_tcp_returns,
+ forever_deferred=deferred,
+ )
diff --git a/src/treq/test/local_httpbin/test/test_parent.py b/src/treq/test/local_httpbin/test/test_parent.py
new file mode 100644
index 0000000..dd57ca7
--- /dev/null
+++ b/src/treq/test/local_httpbin/test/test_parent.py
@@ -0,0 +1,513 @@
+"""
+Tests for :py:mod:`treq.test.local_httpbin.parent`
+"""
+import attr
+
+import json
+
+import signal
+
+import sys
+
+from twisted.internet import defer
+from twisted.internet.interfaces import (IProcessTransport,
+ IReactorCore,
+ IReactorProcess)
+from twisted.python.failure import Failure
+
+from treq.test.util import skip_on_windows_because_of_199
+
+from twisted.internet.error import ProcessTerminated, ConnectionDone
+
+from twisted.test.proto_helpers import MemoryReactor, StringTransport
+from twisted.trial.unittest import SynchronousTestCase
+
+from zope.interface import implementer, verify
+
+from .. import parent, shared
+
+
+skip = skip_on_windows_because_of_199()
+
+
+@attr.s
+class FakeProcessTransportState(object):
+ """
+ State for :py:class:`FakeProcessTransport`.
+ """
+ standard_in_closed = attr.ib(default=False)
+ standard_out_closed = attr.ib(default=False)
+ standard_error_closed = attr.ib(default=False)
+ signals = attr.ib(default=attr.Factory(list))
+
+
+@implementer(IProcessTransport)
+@attr.s
+class FakeProcessTransport(StringTransport, object):
+ """
+ A fake process transport.
+ """
+ pid = 1234
+
+ _state = attr.ib()
+
+ def closeStdin(self):
+ """
+ Close standard in.
+ """
+ self._state.standard_in_closed = True
+
+ def closeStdout(self):
+ """
+ Close standard out.
+ """
+ self._state.standard_out_closed = True
+
+ def closeStderr(self):
+ """
+ Close standard error.
+ """
+ self._state.standard_error_closed = True
+
+ def closeChildFD(self, descriptor):
+ """
+ Close a child's file descriptor.
+
+ :param descriptor: See
+ :py:class:`IProcessProtocol.closeChildFD`
+ """
+
+ def writeToChild(self, childFD, data):
+ """
+ Write data to a child's file descriptor.
+
+ :param childFD: See :py:class:`IProcessProtocol.writeToChild`
+ :param data: See :py:class:`IProcessProtocol.writeToChild`
+ """
+
+ def signalProcess(self, signalID):
+ """
+ Send a signal.
+
+ :param signalID: See
+ :py:class:`IProcessProtocol.signalProcess`
+ """
+ self._state.signals.append(signalID)
+
+
+class FakeProcessTransportTests(SynchronousTestCase):
+ """
+ Tests for :py:class:`FakeProcessTransport`.
+ """
+
+ def setUp(self):
+ self.state = FakeProcessTransportState()
+ self.transport = FakeProcessTransport(self.state)
+
+ def test_provides_interface(self):
+ """
+ Instances provide :py:class:`IProcessTransport`.
+ """
+ verify.verifyObject(IProcessTransport, self.transport)
+
+ def test_closeStdin(self):
+ """
+ Closing standard in updates the state instance.
+ """
+ self.assertFalse(self.state.standard_in_closed)
+ self.transport.closeStdin()
+ self.assertTrue(self.state.standard_in_closed)
+
+ def test_closeStdout(self):
+ """
+ Closing standard out updates the state instance.
+ """
+ self.assertFalse(self.state.standard_out_closed)
+ self.transport.closeStdout()
+ self.assertTrue(self.state.standard_out_closed)
+
+ def test_closeStderr(self):
+ """
+ Closing standard error updates the state instance.
+ """
+ self.assertFalse(self.state.standard_error_closed)
+ self.transport.closeStderr()
+ self.assertTrue(self.state.standard_error_closed)
+
+
+class HTTPServerProcessProtocolTests(SynchronousTestCase):
+ """
+ Tests for :py:class:`parent._HTTPBinServerProcessProtocol`
+ """
+
+ def setUp(self):
+ self.transport_state = FakeProcessTransportState()
+ self.transport = FakeProcessTransport(self.transport_state)
+
+ self.all_data_received = defer.Deferred()
+ self.terminated = defer.Deferred()
+
+ self.protocol = parent._HTTPBinServerProcessProtocol(
+ all_data_received=self.all_data_received,
+ terminated=self.terminated,
+ )
+
+ self.protocol.makeConnection(self.transport)
+
+ def assertStandardInputAndOutputClosed(self):
+ """
+ The transport's standard in, out, and error are closed.
+ """
+ self.assertTrue(self.transport_state.standard_in_closed)
+ self.assertTrue(self.transport_state.standard_out_closed)
+ self.assertTrue(self.transport_state.standard_error_closed)
+
+ def test_receive_http_description(self):
+ """
+ Receiving a serialized :py:class:`_HTTPBinDescription` fires the
+ ``all_data_received`` :py:class:`Deferred`.
+ """
+ self.assertNoResult(self.all_data_received)
+
+ description = shared._HTTPBinDescription("host", 1234, "cert")
+
+ self.protocol.lineReceived(
+ json.dumps(attr.asdict(description)).encode('ascii')
+ )
+
+ self.assertStandardInputAndOutputClosed()
+
+ self.assertEqual(self.successResultOf(self.all_data_received),
+ description)
+
+ def test_receive_unexpected_line(self):
+ """
+ Receiving a line after the description synchronously raises in
+ :py:class:`RuntimeError`
+ """
+ self.test_receive_http_description()
+ with self.assertRaises(RuntimeError):
+ self.protocol.lineReceived(b"unexpected")
+
+ def test_connection_lost_before_receiving_data(self):
+ """
+ If the process terminates before its data is received, both
+ ``all_data_received`` and ``terminated`` errback.
+ """
+ self.assertNoResult(self.all_data_received)
+
+ self.protocol.connectionLost(Failure(ConnectionDone("done")))
+
+ self.assertIsInstance(
+ self.failureResultOf(self.all_data_received).value,
+ ConnectionDone,
+ )
+
+ self.assertIsInstance(
+ self.failureResultOf(self.terminated).value,
+ ConnectionDone,
+ )
+
+ def test_connection_lost(self):
+ """
+ ``terminated`` fires when the connection is lost.
+ """
+ self.test_receive_http_description()
+
+ self.protocol.connectionLost(Failure(ConnectionDone("done")))
+
+ self.assertIsInstance(
+ self.failureResultOf(self.terminated).value,
+ ConnectionDone,
+ )
+
+
+@attr.s
+class SpawnedProcess(object):
+ """
+ A call to :py:class:`MemoryProcessReactor.spawnProcess`.
+ """
+ process_protocol = attr.ib()
+ executable = attr.ib()
+ args = attr.ib()
+ env = attr.ib()
+ path = attr.ib()
+ uid = attr.ib()
+ gid = attr.ib()
+ use_pty = attr.ib()
+ child_fds = attr.ib()
+ returned_process_transport = attr.ib()
+ returned_process_transport_state = attr.ib()
+
+ def send_stdout(self, data):
+ """
+ Send data from the process' standard out.
+
+ :param data: The standard out data.
+ """
+ self.process_protocol.childDataReceived(1, data)
+
+ def end_process(self, reason):
+ """
+ End the process.
+
+ :param reason: The reason.
+ :type reason: :py:class:`Failure`
+ """
+ self.process_protocol.processEnded(reason)
+
+
+@implementer(IReactorCore, IReactorProcess)
+class MemoryProcessReactor(MemoryReactor):
+ """
+ A fake :py:class:`IReactorProcess` and :py:class:`IReactorCore`
+ provider to be used in tests.
+ """
+ def __init__(self):
+ MemoryReactor.__init__(self)
+ self.spawnedProcesses = []
+
+ def spawnProcess(self, processProtocol, executable, args=(), env={},
+ path=None, uid=None, gid=None, usePTY=0, childFDs=None):
+ """
+ :ivar process_protocol: Stores the protocol passed to the reactor.
+ :return: An L{IProcessTransport} provider.
+ """
+ transport_state = FakeProcessTransportState()
+ transport = FakeProcessTransport(transport_state)
+
+ self.spawnedProcesses.append(SpawnedProcess(
+ process_protocol=processProtocol,
+ executable=executable,
+ args=args,
+ env=env,
+ path=path,
+ uid=uid,
+ gid=gid,
+ use_pty=usePTY,
+ child_fds=childFDs,
+ returned_process_transport=transport,
+ returned_process_transport_state=transport_state,
+ ))
+
+ processProtocol.makeConnection(transport)
+
+ return transport
+
+
+class MemoryProcessReactorTests(SynchronousTestCase):
+ """
+ Tests for :py:class:`MemoryProcessReactor`
+ """
+
+ def test_provides_interfaces(self):
+ """
+ :py:class:`MemoryProcessReactor` instances provide
+ :py:class:`IReactorCore` and :py:class:`IReactorProcess`.
+ """
+ reactor = MemoryProcessReactor()
+ verify.verifyObject(IReactorCore, reactor)
+ verify.verifyObject(IReactorProcess, reactor)
+
+
+class HTTPBinProcessTests(SynchronousTestCase):
+ """
+ Tests for :py:class:`_HTTPBinProcesss`.
+ """
+
+ def setUp(self):
+ self.reactor = MemoryProcessReactor()
+ self.opened_file_descriptors = []
+
+ def fd_recording_open(self, *args, **kwargs):
+ """
+ Record the file descriptors of files opened by
+ :py:func:`open`.
+
+ :return: A file object.
+ """
+ fobj = open(*args, **kwargs)
+ self.opened_file_descriptors.append(fobj.fileno())
+ return fobj
+
+ def spawned_process(self):
+ """
+ Assert that ``self.reactor`` has spawned only one process and
+ return the :py:class:`SpawnedProcess` representing it.
+
+ :return: The :py:class:`SpawnedProcess`.
+ """
+ self.assertEqual(len(self.reactor.spawnedProcesses), 1)
+ return self.reactor.spawnedProcesses[0]
+
+ def assertSpawnAndDescription(self, process, args, description):
+ """
+ Assert that spawning the given process invokes the command
+ with the given args, that standard error is redirected, that
+ it is killed at reactor shutdown, and that it returns a
+ description that matches the provided one.
+
+ :param process: :py:class:`_HTTPBinProcesss` instance.
+ :param args: The arguments with which to execute the child
+ process.
+ :type args: :py:class:`tuple` of :py:class:`str`
+
+ :param description: The expected
+ :py:class:`_HTTPBinDescription`.
+
+ :return: The returned :py:class:`_HTTPBinDescription`
+ """
+ process._open = self.fd_recording_open
+
+ description_deferred = process.server_description(self.reactor)
+
+ spawned_process = self.spawned_process()
+
+ self.assertEqual(spawned_process.args, args)
+
+ self.assertEqual(len(self.opened_file_descriptors), 1)
+ [error_log_fd] = self.opened_file_descriptors
+
+ self.assertEqual(spawned_process.child_fds.get(2), error_log_fd)
+
+ self.assertNoResult(description_deferred)
+
+ spawned_process.send_stdout(description.to_json_bytes() + b'\n')
+
+ before_shutdown = self.reactor.triggers["before"]["shutdown"]
+ self.assertEqual(len(before_shutdown), 1)
+ [(before_shutdown_function, _, _)] = before_shutdown
+
+ self.assertEqual(before_shutdown_function, process.kill)
+
+ self.assertEqual(self.successResultOf(description_deferred),
+ description)
+
+ def test_server_description_spawns_process(self):
+ """
+ :py:class:`_HTTPBinProcess.server_description` spawns an
+ ``httpbin`` child process that it monitors with
+ :py:class:`_HTTPBinServerProcessProtocol`, and redirects its
+ standard error to a log file.
+ """
+ httpbin_process = parent._HTTPBinProcess(https=False)
+ description = shared._HTTPBinDescription(host="host", port=1234)
+
+ self.assertSpawnAndDescription(
+ httpbin_process,
+ [
+ sys.executable,
+ '-m',
+ 'treq.test.local_httpbin.child'
+ ],
+ description)
+
+ def test_server_description_spawns_process_https(self):
+ """
+ :py:class:`_HTTPBinProcess.server_description` spawns an
+ ``httpbin`` child process that listens over HTTPS, that it
+ monitors with :py:class:`_HTTPBinServerProcessProtocol`, and
+ redirects the process' standard error to a log file.
+ """
+ httpbin_process = parent._HTTPBinProcess(https=True)
+ description = shared._HTTPBinDescription(host="host",
+ port=1234,
+ cacert="cert")
+
+ self.assertSpawnAndDescription(
+ httpbin_process,
+ [
+ sys.executable,
+ '-m',
+ 'treq.test.local_httpbin.child',
+ '--https',
+ ],
+ description)
+
+ def test_server_description_caches_description(self):
+ """
+ :py:class:`_HTTPBinProcess.server_description` spawns an
+ ``httpbin`` child process only once, after which it returns a
+ cached :py:class:`_HTTPBinDescription`.
+ """
+ httpbin_process = parent._HTTPBinProcess(https=False)
+
+ description_deferred = httpbin_process.server_description(self.reactor)
+
+ self.spawned_process().send_stdout(
+ shared._HTTPBinDescription(host="host", port=1234).to_json_bytes()
+ + b'\n'
+ )
+
+ description = self.successResultOf(description_deferred)
+
+ cached_description_deferred = httpbin_process.server_description(
+ self.reactor,
+ )
+
+ cached_description = self.successResultOf(cached_description_deferred)
+
+ self.assertIs(description, cached_description)
+
+ def test_kill_before_spawn(self):
+ """
+ Killing a process before it has been spawned has no effect.
+ """
+ parent._HTTPBinProcess(https=False).kill()
+
+ def test_kill(self):
+ """
+ Kill terminates the process as quickly as the platform allows,
+ and the termination failure is suppressed.
+ """
+ httpbin_process = parent._HTTPBinProcess(https=False)
+
+ httpbin_process.server_description(self.reactor)
+
+ spawned_process = self.spawned_process()
+
+ spawned_process.send_stdout(
+ shared._HTTPBinDescription(host="host", port=1234).to_json_bytes()
+ + b'\n'
+ )
+
+ termination_deferred = httpbin_process.kill()
+
+ self.assertEqual(
+ spawned_process.returned_process_transport_state.signals,
+ ['KILL'],
+ )
+
+ spawned_process.end_process(
+ Failure(ProcessTerminated(1, signal=signal.SIGKILL)),
+ )
+
+ self.successResultOf(termination_deferred)
+
+ def test_kill_unexpected_exit(self):
+ """
+ The :py:class:`Deferred` returned by
+ :py:meth:`_HTTPBinProcess.kill` errbacks with the failure when
+ it is not :py:class:`ProcessTerminated`, or its signal does
+ not match the expected signal.
+ """
+ for error in [ProcessTerminated(1, signal=signal.SIGIO),
+ ConnectionDone("Bye")]:
+ httpbin_process = parent._HTTPBinProcess(https=False)
+
+ httpbin_process.server_description(self.reactor)
+
+ spawned_process = self.reactor.spawnedProcesses[-1]
+
+ spawned_process.send_stdout(
+ shared._HTTPBinDescription(host="host",
+ port=1234).to_json_bytes()
+ + b'\n'
+ )
+
+ termination_deferred = httpbin_process.kill()
+
+ spawned_process.end_process(Failure(error))
+
+ self.assertIs(self.failureResultOf(termination_deferred).value,
+ error)
diff --git a/src/treq/test/local_httpbin/test/test_shared.py b/src/treq/test/local_httpbin/test/test_shared.py
new file mode 100644
index 0000000..daac6e1
--- /dev/null
+++ b/src/treq/test/local_httpbin/test/test_shared.py
@@ -0,0 +1,41 @@
+"""
+Tests for :py:mod:`treq.test.local_httpbin.shared`
+"""
+from twisted.trial import unittest
+
+from .. import shared
+
+
+class HTTPBinDescriptionTests(unittest.SynchronousTestCase):
+ """
+ Tests for :py:class:`shared._HTTPBinDescription`
+ """
+
+ def test_round_trip(self):
+ """
+ :py:class:`shared._HTTPBinDescription.from_json_bytes` can
+ deserialize the output of
+ :py:class:`shared._HTTPBinDescription.to_json_bytes`
+ """
+ original = shared._HTTPBinDescription(host="host", port=123)
+ round_tripped = shared._HTTPBinDescription.from_json_bytes(
+ original.to_json_bytes(),
+ )
+
+ self.assertEqual(original, round_tripped)
+
+ def test_round_trip_cacert(self):
+ """
+ :py:class:`shared._HTTPBinDescription.from_json_bytes` can
+ deserialize the output of
+ :py:class:`shared._HTTPBinDescription.to_json_bytes` when
+ ``cacert`` is set.
+ """
+ original = shared._HTTPBinDescription(host="host",
+ port=123,
+ cacert='cacert')
+ round_tripped = shared._HTTPBinDescription.from_json_bytes(
+ original.to_json_bytes(),
+ )
+
+ self.assertEqual(original, round_tripped)
diff --git a/treq/test/test_api.py b/src/treq/test/test_api.py
index a62a0a3..61d1b02 100644
--- a/treq/test/test_api.py
+++ b/src/treq/test/test_api.py
@@ -2,7 +2,8 @@ from __future__ import absolute_import, division
import mock
-from treq.test.util import TestCase
+from twisted.trial.unittest import TestCase
+
import treq
from treq._utils import set_global_pool
diff --git a/treq/test/test_auth.py b/src/treq/test/test_auth.py
index 9b0f755..49afc15 100644
--- a/treq/test/test_auth.py
+++ b/src/treq/test/test_auth.py
@@ -1,9 +1,9 @@
import mock
+from twisted.trial.unittest import TestCase
from twisted.web.client import Agent
from twisted.web.http_headers import Headers
-from treq.test.util import TestCase
from treq.auth import _RequestHeaderSettingAgent, add_auth, UnknownAuthConfig
diff --git a/treq/test/test_client.py b/src/treq/test/test_client.py
index 429e29b..eed2e55 100644
--- a/treq/test/test_client.py
+++ b/src/treq/test/test_client.py
@@ -1,3 +1,4 @@
+# -*- encoding: utf-8 -*-
from io import BytesIO
import mock
@@ -7,10 +8,12 @@ from twisted.internet.protocol import Protocol
from twisted.python.failure import Failure
-from twisted.web.client import Agent
+from twisted.trial.unittest import TestCase
+
+from twisted.web.client import Agent, ResponseFailed
from twisted.web.http_headers import Headers
-from treq.test.util import TestCase, with_clock
+from treq.test.util import with_clock
from treq.client import (
HTTPClient, _BodyBufferingProtocol, _BufferedResponse
@@ -34,6 +37,18 @@ class HTTPClientTests(TestCase):
body = self.FileBodyProducer.mock_calls[0][1][0]
self.assertEqual(body.read(), expected)
+ def test_post(self):
+ self.client.post('http://example.com/')
+ self.agent.request.assert_called_once_with(
+ b'POST', b'http://example.com/',
+ Headers({b'accept-encoding': [b'gzip']}), None)
+
+ def test_request_uri_idn(self):
+ self.client.request('GET', u'http://č.net')
+ self.agent.request.assert_called_once_with(
+ b'GET', b'http://xn--bea.net',
+ Headers({b'accept-encoding': [b'gzip']}), None)
+
def test_request_case_insensitive_methods(self):
self.client.request('gEt', 'http://example.com/')
self.agent.request.assert_called_once_with(
@@ -132,6 +147,60 @@ class HTTPClientTests(TestCase):
self.assertBody(b'hello')
+ def test_request_json_dict(self):
+ self.client.request('POST', 'http://example.com/', json={'foo': 'bar'})
+ self.agent.request.assert_called_once_with(
+ b'POST', b'http://example.com/',
+ Headers({b'Content-Type': [b'application/json; charset=UTF-8'],
+ b'accept-encoding': [b'gzip']}),
+ self.FileBodyProducer.return_value)
+ self.assertBody(b'{"foo":"bar"}')
+
+ def test_request_json_tuple(self):
+ self.client.request('POST', 'http://example.com/', json=('foo', 1))
+ self.agent.request.assert_called_once_with(
+ b'POST', b'http://example.com/',
+ Headers({b'Content-Type': [b'application/json; charset=UTF-8'],
+ b'accept-encoding': [b'gzip']}),
+ self.FileBodyProducer.return_value)
+ self.assertBody(b'["foo",1]')
+
+ def test_request_json_number(self):
+ self.client.request('POST', 'http://example.com/', json=1.)
+ self.agent.request.assert_called_once_with(
+ b'POST', b'http://example.com/',
+ Headers({b'Content-Type': [b'application/json; charset=UTF-8'],
+ b'accept-encoding': [b'gzip']}),
+ self.FileBodyProducer.return_value)
+ self.assertBody(b'1.0')
+
+ def test_request_json_string(self):
+ self.client.request('POST', 'http://example.com/', json='hello')
+ self.agent.request.assert_called_once_with(
+ b'POST', b'http://example.com/',
+ Headers({b'Content-Type': [b'application/json; charset=UTF-8'],
+ b'accept-encoding': [b'gzip']}),
+ self.FileBodyProducer.return_value)
+ self.assertBody(b'"hello"')
+
+ def test_request_json_bool(self):
+ self.client.request('POST', 'http://example.com/', json=True)
+ self.agent.request.assert_called_once_with(
+ b'POST', b'http://example.com/',
+ Headers({b'Content-Type': [b'application/json; charset=UTF-8'],
+ b'accept-encoding': [b'gzip']}),
+ self.FileBodyProducer.return_value)
+ self.assertBody(b'true')
+
+ def test_request_json_none(self):
+ self.client.request('POST', 'http://example.com/', json=None)
+ self.agent.request.assert_called_once_with(
+ b'POST', b'http://example.com/',
+ Headers({b'Content-Type': [b'application/json; charset=UTF-8'],
+ b'accept-encoding': [b'gzip']}),
+ self.FileBodyProducer.return_value)
+ self.assertBody(b'null')
+
@mock.patch('treq.client.uuid.uuid4', mock.Mock(return_value="heyDavid"))
def test_request_no_name_attachment(self):
@@ -330,6 +399,27 @@ class HTTPClientTests(TestCase):
# YOLO public attribute.
self.assertEqual(self.successResultOf(d).original, response)
+ def test_request_post_redirect_denied(self):
+ response = mock.Mock(code=302, headers=Headers({'Location': ['/']}))
+ self.agent.request.return_value = succeed(response)
+ d = self.client.post('http://www.example.com')
+ self.failureResultOf(d, ResponseFailed)
+
+ def test_request_browser_like_redirects(self):
+ response = mock.Mock(code=302, headers=Headers({'Location': ['/']}))
+
+ self.agent.request.return_value = succeed(response)
+
+ raw = mock.Mock(return_value=[])
+ final_resp = mock.Mock(code=200, headers=mock.Mock(getRawHeaders=raw))
+ with mock.patch('twisted.web.client.RedirectAgent._handleRedirect',
+ return_value=final_resp):
+ d = self.client.post('http://www.google.com',
+ browser_like_redirects=True,
+ unbuffered=True)
+
+ self.assertEqual(self.successResultOf(d).original, final_resp)
+
class BodyBufferingProtocolTests(TestCase):
def test_buffers_data(self):
diff --git a/treq/test/test_content.py b/src/treq/test/test_content.py
index 287375b..60cd998 100644
--- a/treq/test/test_content.py
+++ b/src/treq/test/test_content.py
@@ -1,13 +1,13 @@
+# -*- coding: utf-8 -*-
import mock
from twisted.python.failure import Failure
+from twisted.trial.unittest import TestCase
from twisted.web.http_headers import Headers
from twisted.web.client import ResponseDone, ResponseFailed
from twisted.web.http import PotentialDataLoss
-from treq.test.util import TestCase
-
from treq import collect, content, json_content, text_content
from treq.client import _BufferedResponse
@@ -126,6 +126,36 @@ class ContentTests(TestCase):
self.assertEqual(self.successResultOf(d), {"msg": "hello!"})
+ def test_json_content_unicode(self):
+ """
+ When Unicode JSON content is received, the JSON text should be
+ correctly decoded.
+ RFC7159 (8.1): "JSON text SHALL be encoded in UTF-8, UTF-16, or UTF-32.
+ The default encoding is UTF-8"
+ """
+ self.response.headers = Headers()
+ d = json_content(self.response)
+
+ self.protocol.dataReceived(u'{"msg":"hëlló!"}'.encode('utf-8'))
+ self.protocol.connectionLost(Failure(ResponseDone()))
+
+ self.assertEqual(self.successResultOf(d), {u'msg': u'hëlló!'})
+
+ def test_json_content_utf16(self):
+ """
+ JSON received is decoded according to the charset given in the
+ Content-Type header.
+ """
+ self.response.headers = Headers({
+ b'Content-Type': [b"application/json; charset='UTF-16LE'"],
+ })
+ d = json_content(self.response)
+
+ self.protocol.dataReceived(u'{"msg":"hëlló!"}'.encode('UTF-16LE'))
+ self.protocol.connectionLost(Failure(ResponseDone()))
+
+ self.assertEqual(self.successResultOf(d), {u'msg': u'hëlló!'})
+
def test_text_content(self):
self.response.headers = Headers(
{b'Content-Type': [b'text/plain; charset=utf-8']})
@@ -157,3 +187,32 @@ class ContentTests(TestCase):
self.protocol.connectionLost(Failure(ResponseDone()))
self.assertEqual(self.successResultOf(d), u'\xa1')
+
+ def test_content_application_json_default_encoding(self):
+ self.response.headers = Headers(
+ {b'Content-Type': [b'application/json']})
+
+ d = text_content(self.response)
+
+ self.protocol.dataReceived(b'gr\xc3\xbcn')
+ self.protocol.connectionLost(Failure(ResponseDone()))
+
+ self.assertEqual(self.successResultOf(d), u'grün')
+
+ def test_text_content_unicode_headers(self):
+ """
+ Header parsing is robust against unicode header names and values.
+ """
+ self.response.headers = Headers({
+ b'Content-Type': [
+ u'text/plain; charset="UTF-16BE"; u=ᛃ'.encode('utf-8')],
+ u'Coördination'.encode('iso-8859-1'): [
+ u'koʊˌɔrdɪˈneɪʃən'.encode('utf-8')],
+ })
+
+ d = text_content(self.response)
+
+ self.protocol.dataReceived(u'ᚠᚡ'.encode('UTF-16BE'))
+ self.protocol.connectionLost(Failure(ResponseDone()))
+
+ self.assertEqual(self.successResultOf(d), u'ᚠᚡ')
diff --git a/treq/test/test_multipart.py b/src/treq/test/test_multipart.py
index 4baf603..4aece22 100644
--- a/treq/test/test_multipart.py
+++ b/src/treq/test/test_multipart.py
@@ -9,7 +9,7 @@ from io import BytesIO
from twisted.trial import unittest
from zope.interface.verify import verifyObject
-from twisted.python import failure, compat
+from twisted.python import compat
from twisted.internet import task
from twisted.web.client import FileBodyProducer
from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer
@@ -46,51 +46,6 @@ class MultiPartProducerTestCase(unittest.TestCase):
self.cooperator = task.Cooperator(
self._termination, self._scheduled.append)
- def successResultOf(self, deferred):
- """
- Backport from 13.0 for compatibility with older Twisted versions
- """
- result = []
- deferred.addBoth(result.append)
- if not result:
- self.fail(
- "Success result expected on %r, found no result instead" % (
- deferred,))
- elif isinstance(result[0], failure.Failure):
- self.fail(
- "Success result expected on %r, "
- "found failure result (%r) instead" % (deferred, result[0]))
- else:
- return result[0]
-
- def assertNoResult(self, deferred):
- """
- Backport from 13.0 for compatibility with older Twisted versions
- """
- result = []
- deferred.addBoth(result.append)
- if result:
- self.fail(
- "No result expected on %r, found %r instead" % (
- deferred, result[0]))
-
- def failureResultOf(self, deferred):
- """
- Backport from 13.0 for compatibility with older Twisted versions
- """
- result = []
- deferred.addBoth(result.append)
- if not result:
- self.fail(
- "Failure result expected on %r, found no result instead" % (
- deferred,))
- elif not isinstance(result[0], failure.Failure):
- self.fail(
- "Failure result expected on %r, "
- "found success result (%r) instead" % (deferred, result[0]))
- else:
- return result[0]
-
def getOutput(self, producer, with_producer=False):
"""
A convenience function to consume and return outpute.
diff --git a/src/treq/test/test_response.py b/src/treq/test/test_response.py
new file mode 100644
index 0000000..e63bc58
--- /dev/null
+++ b/src/treq/test/test_response.py
@@ -0,0 +1,139 @@
+from decimal import Decimal
+
+from twisted.trial.unittest import SynchronousTestCase
+
+from twisted.python.failure import Failure
+from twisted.web.client import ResponseDone
+from twisted.web.iweb import UNKNOWN_LENGTH
+from twisted.web.http_headers import Headers
+
+from treq.response import _Response
+
+
+class FakeResponse(object):
+ def __init__(self, code, headers, body=()):
+ self.code = code
+ self.headers = headers
+ self.previousResponse = None
+ self._body = body
+ self.length = sum(len(c) for c in body)
+
+ def setPreviousResponse(self, response):
+ self.previousResponse = response
+
+ def deliverBody(self, protocol):
+ for chunk in self._body:
+ protocol.dataReceived(chunk)
+ protocol.connectionLost(Failure(ResponseDone()))
+
+
+class ResponseTests(SynchronousTestCase):
+ def test_repr_content_type(self):
+ """
+ When the response has a Content-Type header its value is included in
+ the response.
+ """
+ headers = Headers({'Content-Type': ['text/html']})
+ original = FakeResponse(200, headers, body=[b'<!DOCTYPE html>'])
+ self.assertEqual(
+ "<treq.response._Response 200 'text/html' 15 bytes>",
+ repr(_Response(original, None)),
+ )
+
+ def test_repr_content_type_missing(self):
+ """
+ A request with no Content-Type just displays an empty field.
+ """
+ original = FakeResponse(204, Headers(), body=[b''])
+ self.assertEqual(
+ "<treq.response._Response 204 '' 0 bytes>",
+ repr(_Response(original, None)),
+ )
+
+ def test_repr_content_type_hostile(self):
+ """
+ Garbage in the Content-Type still produces a reasonable representation.
+ """
+ headers = Headers({'Content-Type': [u'\u2e18', ' x/y']})
+ original = FakeResponse(418, headers, body=[b''])
+ self.assertEqual(
+ r"<treq.response._Response 418 '\xe2\xb8\x98, x/y' 0 bytes>",
+ repr(_Response(original, None)),
+ )
+
+ def test_repr_unknown_length(self):
+ """
+ A HTTP 1.0 or chunked response displays an unknown length.
+ """
+ headers = Headers({'Content-Type': ['text/event-stream']})
+ original = FakeResponse(200, headers)
+ original.length = UNKNOWN_LENGTH
+ self.assertEqual(
+ "<treq.response._Response 200 'text/event-stream' unknown size>",
+ repr(_Response(original, None)),
+ )
+
+ def test_collect(self):
+ original = FakeResponse(200, Headers(), body=[b'foo', b'bar', b'baz'])
+ calls = []
+ _Response(original, None).collect(calls.append)
+ self.assertEqual([b'foo', b'bar', b'baz'], calls)
+
+ def test_content(self):
+ original = FakeResponse(200, Headers(), body=[b'foo', b'bar', b'baz'])
+ self.assertEqual(
+ b'foobarbaz',
+ self.successResultOf(_Response(original, None).content()),
+ )
+
+ def test_json(self):
+ original = FakeResponse(200, Headers(), body=[b'{"foo": ', b'"bar"}'])
+ self.assertEqual(
+ {'foo': 'bar'},
+ self.successResultOf(_Response(original, None).json()),
+ )
+
+ def test_json_customized(self):
+ original = FakeResponse(200, Headers(), body=[b'{"foo": ',
+ b'1.0000000000000001}'])
+ self.assertEqual(
+ self.successResultOf(_Response(original, None).json(
+ parse_float=Decimal)
+ )["foo"],
+ Decimal("1.0000000000000001")
+ )
+
+ def test_text(self):
+ headers = Headers({b'content-type': [b'text/plain;charset=utf-8']})
+ original = FakeResponse(200, headers, body=[b'\xe2\x98', b'\x83'])
+ self.assertEqual(
+ u'\u2603',
+ self.successResultOf(_Response(original, None).text()),
+ )
+
+ def test_history(self):
+ redirect1 = FakeResponse(
+ 301,
+ Headers({'location': ['http://example.com/']})
+ )
+
+ redirect2 = FakeResponse(
+ 302,
+ Headers({'location': ['https://example.com/']})
+ )
+ redirect2.setPreviousResponse(redirect1)
+
+ final = FakeResponse(200, Headers({}))
+ final.setPreviousResponse(redirect2)
+
+ wrapper = _Response(final, None)
+
+ history = wrapper.history()
+
+ self.assertEqual(wrapper.code, 200)
+ self.assertEqual(history[0].code, 301)
+ self.assertEqual(history[1].code, 302)
+
+ def test_no_history(self):
+ wrapper = _Response(FakeResponse(200, Headers({})), None)
+ self.assertEqual(wrapper.history(), [])
diff --git a/treq/test/test_testing.py b/src/treq/test/test_testing.py
index 5ddb7a9..6a13881 100644
--- a/treq/test/test_testing.py
+++ b/src/treq/test/test_testing.py
@@ -8,6 +8,7 @@ from mock import ANY
from six import text_type, binary_type
+from twisted.trial.unittest import TestCase
from twisted.web.client import ResponseFailed
from twisted.web.error import SchemeNotSupported
from twisted.web.resource import Resource
@@ -16,7 +17,6 @@ from twisted.python.compat import _PY3
import treq
-from treq.test.util import TestCase
from treq.testing import (
HasHeaders,
RequestSequence,
@@ -294,6 +294,15 @@ class HasHeadersTests(TestCase):
"""
self.assertNotEqual(HasHeaders({b'a': [b'a']}), {b'a': [b'A']})
+ def test_bytes_encoded_forms(self):
+ """
+ The :obj:`HasHeaders` equality function compares the bytes-encoded
+ forms of both sets of headers.
+ """
+ self.assertEqual(HasHeaders({b'a': [b'a']}), {u'a': [u'a']})
+
+ self.assertEqual(HasHeaders({u'b': [u'b']}), {b'b': [b'b']})
+
def test_repr(self):
"""
:obj:`HasHeaders` returns a nice string repr.
@@ -400,6 +409,8 @@ class RequestSequenceTests(TestCase):
d = stub.get('https://anything', data=b'what', headers={b'1': b'1'})
resp = self.successResultOf(d)
self.assertEqual(500, resp.code)
+ self.assertEqual(b'StubbingError',
+ self.successResultOf(resp.content()))
self.assertEqual(1, len(self.async_failures))
self.assertIn("No more requests expected, but request",
self.async_failures[0])
@@ -455,3 +466,17 @@ class RequestSequenceTests(TestCase):
# no asynchronous failures (mismatches, etc.)
self.assertEqual([], self.async_failures)
+
+ def test_async_failures_logged(self):
+ """
+ When no `async_failure_reporter` is passed async failures are logged by
+ default.
+ """
+ sequence = RequestSequence([])
+ stub = StubTreq(StringStubbingResource(sequence))
+
+ with sequence.consume(self.fail):
+ self.successResultOf(stub.get('https://example.com'))
+
+ [failure] = self.flushLoggedErrors()
+ self.assertIsInstance(failure.value, AssertionError)
diff --git a/treq/test/test_treq_integration.py b/src/treq/test/test_treq_integration.py
index 6b7db81..14b81da 100644
--- a/treq/test/test_treq_integration.py
+++ b/src/treq/test/test_treq_integration.py
@@ -1,32 +1,25 @@
from io import BytesIO
+from twisted.python.url import URL
+
from twisted.trial.unittest import TestCase
from twisted.internet.defer import CancelledError, inlineCallbacks
from twisted.internet.task import deferLater
from twisted.internet import reactor
from twisted.internet.tcp import Client
+from twisted.internet.ssl import Certificate, trustRootFromCertificates
-from twisted import version as current_version
-from twisted.python.versions import Version
+from twisted.web.client import (Agent, BrowserLikePolicyForHTTPS,
+ HTTPConnectionPool, ResponseFailed)
-from twisted.web.client import HTTPConnectionPool, ResponseFailed
+from treq.test.util import DEBUG, skip_on_windows_because_of_199
-from treq.test.util import DEBUG
+from .local_httpbin.parent import _HTTPBinProcess
import treq
-HTTPBIN_URL = "http://httpbin.org"
-HTTPSBIN_URL = "https://httpbin.org"
-
-
-def todo_relative_redirect(test_method):
- expected_version = Version('twisted', 13, 1, 0)
- if current_version < expected_version:
- test_method.todo = (
- "Relative Redirects are not supported in Twisted versions "
- "prior to: {0}").format(expected_version.short())
- return test_method
+skip = skip_on_windows_because_of_199()
@inlineCallbacks
@@ -43,13 +36,16 @@ def print_response(response):
def with_baseurl(method):
def _request(self, url, *args, **kwargs):
- return method(self.baseurl + url, *args, pool=self.pool, **kwargs)
+ return method(self.baseurl + url,
+ *args,
+ agent=self.agent,
+ pool=self.pool,
+ **kwargs)
return _request
class TreqIntegrationTests(TestCase):
- baseurl = HTTPBIN_URL
get = with_baseurl(treq.get)
head = with_baseurl(treq.head)
post = with_baseurl(treq.post)
@@ -57,7 +53,16 @@ class TreqIntegrationTests(TestCase):
patch = with_baseurl(treq.patch)
delete = with_baseurl(treq.delete)
+ _httpbin_process = _HTTPBinProcess(https=False)
+
+ @inlineCallbacks
def setUp(self):
+ description = yield self._httpbin_process.server_description(
+ reactor)
+ self.baseurl = URL(scheme=u"http",
+ host=description.host,
+ port=description.port).asText()
+ self.agent = Agent(reactor)
self.pool = HTTPConnectionPool(reactor, False)
def tearDown(self):
@@ -112,7 +117,6 @@ class TreqIntegrationTests(TestCase):
self.assertEqual(response.code, 200)
yield print_response(response)
- @todo_relative_redirect
@inlineCallbacks
def test_get_302_relative_redirect(self):
response = yield self.get('/relative-redirect/1')
@@ -139,7 +143,6 @@ class TreqIntegrationTests(TestCase):
self.assertEqual(response.code, 200)
yield print_response(response)
- @todo_relative_redirect
@inlineCallbacks
def test_head_302_relative_redirect(self):
response = yield self.head('/relative-redirect/1')
@@ -265,4 +268,22 @@ class TreqIntegrationTests(TestCase):
class HTTPSTreqIntegrationTests(TreqIntegrationTests):
- baseurl = HTTPSBIN_URL
+ _httpbin_process = _HTTPBinProcess(https=True)
+
+ @inlineCallbacks
+ def setUp(self):
+ description = yield self._httpbin_process.server_description(
+ reactor)
+ self.baseurl = URL(scheme=u"https",
+ host=description.host,
+ port=description.port).asText()
+
+ root = trustRootFromCertificates(
+ [Certificate.loadPEM(description.cacert)],
+ )
+ self.agent = Agent(
+ reactor,
+ contextFactory=BrowserLikePolicyForHTTPS(root),
+ )
+
+ self.pool = HTTPConnectionPool(reactor, False)
diff --git a/treq/test/test_utils.py b/src/treq/test/test_utils.py
index 236bfe0..2023ddc 100644
--- a/treq/test/test_utils.py
+++ b/src/treq/test/test_utils.py
@@ -1,6 +1,6 @@
import mock
-from treq.test.util import TestCase
+from twisted.trial.unittest import TestCase
from treq._utils import default_reactor, default_pool, set_global_pool
diff --git a/src/treq/test/util.py b/src/treq/test/util.py
new file mode 100644
index 0000000..16f82c2
--- /dev/null
+++ b/src/treq/test/util.py
@@ -0,0 +1,31 @@
+import os
+import platform
+
+import mock
+
+from twisted.internet import reactor
+from twisted.internet.task import Clock
+
+DEBUG = os.getenv("TREQ_DEBUG", False) == "true"
+
+is_pypy = platform.python_implementation() == 'PyPy'
+
+
+def with_clock(fn):
+ def wrapper(*args, **kwargs):
+ clock = Clock()
+ with mock.patch.object(reactor, 'callLater', clock.callLater):
+ return fn(*(args + (clock,)), **kwargs)
+ return wrapper
+
+
+def skip_on_windows_because_of_199():
+ """
+ Return a skip describing issue #199 under Windows.
+
+ :return: A :py:class:`str` skip reason.
+ """
+ if platform.system() == 'Windows':
+ return ("HTTPBin process cannot run under Windows."
+ " See https://github.com/twisted/treq/issues/199")
+ return None
diff --git a/treq/testing.py b/src/treq/testing.py
index 4d45894..c877be1 100644
--- a/treq/testing.py
+++ b/src/treq/testing.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
"""
In-memory version of treq for testing.
"""
@@ -16,33 +17,77 @@ from twisted.internet.address import IPv4Address
from twisted.internet.defer import succeed
from twisted.internet.interfaces import ISSLTransport
+from twisted.logger import Logger
+
+from twisted.python.failure import Failure
from twisted.python.urlpath import URLPath
+from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.web.client import Agent
+from twisted.web.error import SchemeNotSupported
+from twisted.web.iweb import IAgent, IAgentEndpointFactory, IBodyProducer
from twisted.web.resource import Resource
from twisted.web.server import Site
-from twisted.web.iweb import IAgent, IBodyProducer
from zope.interface import directlyProvides, implementer
import treq
from treq.client import HTTPClient
+import attr
+
+
+@implementer(IAgentEndpointFactory)
+@attr.s
+class _EndpointFactory(object):
+ """
+ An endpoint factory used by :class:`RequestTraversalAgent`.
+
+ :ivar reactor: The agent's reactor.
+ :type reactor: :class:`MemoryReactor`
+ """
+
+ reactor = attr.ib()
+
+ def endpointForURI(self, uri):
+ """
+ Create an endpoint that represents an in-memory connection to
+ a URI.
+
+ Note: This always creates a
+ :class:`~twisted.internet.endpoints.TCP4ClientEndpoint` on the
+ assumption :class:`RequestTraversalAgent` ignores everything
+ about the endpoint but its port.
+
+ :param uri: The URI to connect to.
+ :type uri: :class:`~twisted.web.client.URI`
+
+ :return: The endpoint.
+ :rtype: An
+ :class:`~twisted.internet.interfaces.IStreamClientEndpoint`
+ provider.
+ """
+
+ if uri.scheme not in {b'http', b'https'}:
+ raise SchemeNotSupported("Unsupported scheme: %r" % (uri.scheme,))
+ return TCP4ClientEndpoint(self.reactor, "127.0.0.1", uri.port)
@implementer(IAgent)
class RequestTraversalAgent(object):
"""
- :obj:`IAgent` implementation that issues an in-memory request rather than
- going out to a real network socket.
+ :obj:`~twisted.web.iweb.IAgent` implementation that issues an in-memory
+ request rather than going out to a real network socket.
"""
def __init__(self, rootResource):
"""
- :param rootResource: The twisted IResource at the root of the resource
- tree.
+ :param rootResource: The Twisted `IResource` at the root of the
+ resource tree.
"""
self._memoryReactor = MemoryReactor()
- self._realAgent = Agent(reactor=self._memoryReactor)
+ self._realAgent = Agent.usingEndpointFactory(
+ reactor=self._memoryReactor,
+ endpointFactory=_EndpointFactory(self._memoryReactor))
self._rootResource = rootResource
self._pumps = set()
@@ -76,12 +121,8 @@ class RequestTraversalAgent(object):
else:
scheme = URLPath.fromString(uri).scheme
- if scheme == b"https":
- host, port, factory, context_factory, timeout, bindAddress = (
- self._memoryReactor.sslClients[-1])
- else:
- host, port, factory, timeout, bindAddress = (
- self._memoryReactor.tcpClients[-1])
+ host, port, factory, timeout, bindAddress = (
+ self._memoryReactor.tcpClients[-1])
serverAddress = IPv4Address('TCP', '127.0.0.1', port)
clientAddress = IPv4Address('TCP', '127.0.0.1', 31337)
@@ -98,10 +139,6 @@ class RequestTraversalAgent(object):
clientProtocol, isServer=False,
hostAddress=clientAddress, peerAddress=serverAddress)
- # Twisted 13.2 compatibility.
- serverTransport.abortConnection = serverTransport.loseConnection
- clientTransport.abortConnection = clientTransport.loseConnection
-
if scheme == b"https":
# Provide ISSLTransport on both transports, so everyone knows that
# this is HTTPS.
@@ -122,7 +159,7 @@ class RequestTraversalAgent(object):
This is only necessary if a :obj:`Resource` under test returns
:obj:`NOT_DONE_YET` from its ``render`` method, making a response
asynchronous. In that case, after each write from the server,
- :meth:`pump` must be called so the client can see it.
+ :meth:`flush()` must be called so the client can see it.
"""
old_pumps = self._pumps
new_pumps = self._pumps = set()
@@ -143,8 +180,8 @@ class _SynchronousProducer(object):
This does not implement the :func:`IBodyProducer.stopProducing` method,
because that is very difficult to trigger. (The request from
- RequestTraversalAgent would have to be canceled while it is still in the
- transmitting state), and the intent is to use RequestTraversalAgent to
+ `RequestTraversalAgent` would have to be canceled while it is still in the
+ transmitting state), and the intent is to use `RequestTraversalAgent` to
make synchronous requests.
"""
@@ -184,14 +221,15 @@ def _reject_files(f):
class StubTreq(object):
"""
A fake version of the treq module that can be used for testing that
- provides all the function calls exposed in treq.__all__.
-
- :ivar resource: A :obj:`Resource` object that provides the fake responses
+ provides all the function calls exposed in :obj:`treq.__all__`.
"""
def __init__(self, resource):
"""
Construct a client, and pass through client methods and/or
treq.content functions.
+
+ :param resource: A :obj:`Resource` object that provides the fake
+ responses
"""
_agent = RequestTraversalAgent(resource)
_client = HTTPClient(agent=_agent,
@@ -216,30 +254,31 @@ class StringStubbingResource(Resource):
The resource uses the callable to return a real response as a result of a
request.
- The parameters for the callable are::
+ The parameters for the callable are:
+
+ - ``method``, the HTTP method as `bytes`.
+ - ``url``, the full URL of the request as text.
+ - ``params``, a dictionary of query parameters mapping query keys
+ lists of values (sorted alphabetically).
+ - ``headers``, a dictionary of headers mapping header keys to
+ a list of header values (sorted alphabetically).
+ - ``data``, the request body as `bytes`.
- :param bytes method: An HTTP method
- :param bytes url: The full URL of the request
- :param dict params: A dictionary of query parameters mapping query keys
- lists of values (sorted alphabetically)
- :param dict headers: A dictionary of headers mapping header keys to
- a list of header values (sorted alphabetically)
- :param str data: The request body.
- :return: a ``tuple`` of (code, headers, body) where the code is
- the HTTP status code, the headers is a dictionary of bytes
- (unlike the `headers` parameter, which is a dictionary of lists),
- and body is a string that will be returned as the response body.
+ The callable must return a ``tuple`` of (code, headers, body) where the
+ code is the HTTP status code, the headers is a dictionary of bytes (unlike
+ the `headers` parameter, which is a dictionary of lists), and body is
+ a string that will be returned as the response body.
If there is a stubbing error, the return value is undefined (if an
- exception is raised, :obj:`Resource` will just eat it and return 500
- in its place). The callable, or whomever creates the callable, should
- have a way to handle error reporting.
+ exception is raised, :obj:`~twisted.web.resource.Resource` will just eat it
+ and return 500 in its place). The callable, or whomever creates the
+ callable, should have a way to handle error reporting.
"""
isLeaf = True
def __init__(self, get_response_for):
"""
- See ``StringStubbingResource``.
+ See :class:`StringStubbingResource`.
"""
Resource.__init__(self)
self._get_response_for = get_response_for
@@ -289,6 +328,12 @@ def _maybeEncode(someStr):
return someStr
+def _maybeEncodeHeaders(headers):
+ """ Convert a headers mapping to its bytes-encoded form. """
+ return {_maybeEncode(k).lower(): [_maybeEncode(v) for v in vs]
+ for k, vs in headers.items()}
+
+
class HasHeaders(object):
"""
Since Twisted adds headers to a request, such as the host and the content
@@ -298,17 +343,18 @@ class HasHeaders(object):
This wraps a set of headers, and can be used in an equality test against
a superset if the provided headers. The headers keys are lowercased, and
keys and values are compared in their bytes-encoded forms.
+
+ Headers should be provided as a mapping from strings or bytes to a list of
+ strings or bytes.
"""
def __init__(self, headers):
- self._headers = dict([(_maybeEncode(k).lower(), _maybeEncode(v))
- for k, v in headers.items()])
+ self._headers = _maybeEncodeHeaders(headers)
def __repr__(self):
return "HasHeaders({0})".format(repr(self._headers))
def __eq__(self, other_headers):
- compare_to = dict([(_maybeEncode(k).lower(), _maybeEncode(v))
- for k, v in other_headers.items()])
+ compare_to = _maybeEncodeHeaders(other_headers)
return (set(self._headers.keys()).issubset(set(compare_to.keys())) and
all([set(v).issubset(set(compare_to[k]))
@@ -328,44 +374,92 @@ class RequestSequence(object):
...]
Expects the requests to arrive in sequence order. If there are no more
- responses, or the request's paramters do not match the next item's expected
- request paramters, raises :obj:`AssertionError`.
-
- For the expected request arguments::
-
- - ``method`` should be `bytes` normalized to lowercase.
- - ``url`` should be normalized as per the transformations in
- https://en.wikipedia.org/wiki/URL_normalization that (usually) preserve
- semantics. A url to `http://something-that-looks-like-a-directory`
- would be normalized to `http://something-that-looks-like-a-directory/`
- and a url to `http://something-that-looks-like-a-page/page.html`
- remains unchanged.
- - ``params`` is a dictionary mapping `bytes` to `lists` of `bytes`
- - ``headers`` is a dictionary mapping `bytes` to `lists` of `bytes` - note
- that :obj:`twisted.web.client.Agent` may add its own headers though,
- which are not guaranteed (for instance, `user-agent` or
- `content-length`), so it's better to use some kind of matcher like
- :obj:`HasHeaders`.
- - ``data`` is a `bytes`
-
- For the response::
-
- - ``code`` is an integer representing the HTTP status code to return
- - ``headers`` is a dictionary mapping `bytes` to `bytes` or `lists` of
- `bytes`
- - ``body`` is a `bytes`
-
- :ivar list sequence: The sequence of expected request arguments mapped to
- stubbed responses
- :ivar async_failure_reporter: A callable that takes a single message
- reporting failures - it's asynchronous because it cannot just raise
- an exception - if it does, :obj:`Resource.render` will just convert
- that into a 500 response, and there will be no other failure reporting
- mechanism.
+ responses, or the request's parameters do not match the next item's
+ expected request parameters, calls `sync_failure_reporter` or
+ `async_failure_reporter`.
+
+ For the expected request tuples:
+
+ - ``method`` should be :class:`bytes` normalized to lowercase.
+ - ``url`` should be a `str` normalized as per the `transformations in that
+ (usually) preserve semantics
+ <https://en.wikipedia.org/wiki/URL_normalization>`_. A URL to
+ `http://something-that-looks-like-a-directory` would be normalized to
+ `http://something-that-looks-like-a-directory/`
+ and a URL to `http://something-that-looks-like-a-page/page.html`
+ remains unchanged.
+ - ``params`` is a dictionary mapping :class:`bytes` to :class:`list` of
+ :class:`bytes`.
+ - ``headers`` is a dictionary mapping :class:`bytes` to :class:`list` of
+ :class:`bytes` -- note that :class:`twisted.web.client.Agent` may add its
+ own headers which are not guaranteed to be present (for instance,
+ `user-agent` or `content-length`), so it's better to use some kind of
+ matcher like :class:`HasHeaders`.
+ - ``data`` is a :class:`bytes`.
+
+ For the response tuples:
+
+ - ``code`` is an integer representing the HTTP status code to return.
+ - ``headers`` is a dictionary mapping :class:`bytes` to :class:`bytes` or
+ :class:`list` of :class:`bytes`.
+ - ``body`` is a :class:`bytes`.
+
+ :ivar list sequence: A sequence of (request tuple, response tuple)
+ two-tuples, as described above.
+ :ivar async_failure_reporter: An optional callable that takes
+ a :class:`str` message indicating a failure. It's asynchronous because
+ it cannot just raise an exception—if it does, :meth:`Resource.render
+ <twisted.web.resource.Resource.render>` will just convert that into
+ a 500 response, and there will be no other failure reporting mechanism.
+
+ When the `async_failure_reporter` parameter is not passed, async failures
+ will be reported via a :class:`twisted.logger.Logger` instance, which
+ Trial's test case classes (:class:`twisted.trial.unittest.TestCase` and
+ :class:`~twisted.trial.unittest.SynchronousTestCase`) will translate into
+ a test failure.
+
+ .. note::
+
+ Some versions of
+ :class:`twisted.trial.unittest.SynchronousTestCase` report
+ logged errors on the wrong test: see `Twisted #9267
+ <https://twistedmatrix.com/trac/ticket/9267>`_.
+
+ .. TODO Update the above note to say what version of
+ SynchronousTestCase is fixed once Twisted >17.5.0 is released.
+
+ When not subclassing Trial's classes you must pass `async_failure_reporter`
+ and implement equivalent behavior or errors will pass silently. For
+ example::
+
+ async_failures = []
+ sequence_stubs = RequestSequence([...], async_failures.append)
+ stub_treq = StubTreq(StringStubbingResource(sequence_stubs))
+ with sequence_stubs.consume(self.fail): # self = unittest.TestCase
+ stub_treq.get('http://fakeurl.com')
+
+ self.assertEqual([], async_failures)
"""
- def __init__(self, sequence, async_failure_reporter):
+ _log = Logger()
+
+ def __init__(self, sequence, async_failure_reporter=None):
self._sequence = sequence
- self._async_reporter = async_failure_reporter
+ self._async_reporter = async_failure_reporter or self._log_async_error
+
+ def _log_async_error(self, message):
+ """
+ The default async failure reporter—see `async_failure_reporter`. Logs
+ a failure which wraps an :ex:`AssertionError`.
+
+ :param str message: Failure message
+ """
+ # Passing message twice may look redundant, but Trial only preserves
+ # the Failure, not the log message.
+ self._log.failure(
+ "RequestSequence async error: {message}",
+ message=message,
+ failure=Failure(AssertionError(message)),
+ )
def consumed(self):
"""
@@ -382,7 +476,8 @@ class RequestSequence(object):
sequence_stubs = RequestSequence([...])
stub_treq = StubTreq(StringStubbingResource(sequence_stubs))
- with sequence_stubs.consume(self.fail): # self = unittest.TestCase
+ # self = twisted.trial.unittest.SynchronousTestCase
+ with sequence_stubs.consume(self.fail):
stub_treq.get('http://fakeurl.com')
stub_treq.get('http://another-fake-url.com')
@@ -413,7 +508,7 @@ class RequestSequence(object):
self._async_reporter(
"No more requests expected, but request {0!r} made.".format(
(method, url, params, headers, data)))
- return (500, {}, "StubbingError")
+ return (500, {}, b"StubbingError")
expected, response = self._sequence[0]
e_method, e_url, e_params, e_headers, e_data = expected
@@ -433,7 +528,7 @@ class RequestSequence(object):
"\nMismatches: {2!r}"
.format(expected, (method, url, params, headers, data),
mismatches))
- return (500, {}, "StubbingError")
+ return (500, {}, b"StubbingError")
self._sequence = self._sequence[1:]
diff --git a/tox.ini b/tox.ini
index 49b6b88..fbcd424 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,39 +1,40 @@
[tox]
envlist =
- {pypy,py27}-twisted_{14.0,15.1,15.3,15.5}-pyopenssl_{0.13.1,0.14,0.15.1},
- {py33,py34,py35}-twisted_{15.5}-pyopenssl_{0.15.1},
- {pypy,py27,py33,py34,py35}-twisted_trunk-pyopenssl_trunk,
- pypi-readme, check-manifest, flake8
+ {pypy,py27,py34,py35,py36}-twisted_{lowest,latest},
+ {pypy,py27,py34,py35,py36}-twisted_trunk-pyopenssl_trunk,
+ pypi-readme, check-manifest, flake8, docs
[testenv]
+extras = dev
deps =
coverage
mock
- ; Can't use Cryptography 1.0 on older PyPys
- pypy: cryptography<=0.9
- twisted_14.0: twisted==14.0
- twisted_15.1: twisted==15.1
- twisted_15.3: twisted==15.3
- twisted_15.5: twisted==15.5
+ twisted_lowest: Twisted==16.4.0
+ twisted_latest: Twisted
twisted_trunk: https://github.com/twisted/twisted/archive/trunk.zip
- pyopenssl_0.13.1: pyopenssl==0.13.1
- pyopenssl_0.14: pyopenssl==0.14
- pyopenssl_0.15.1: pyopenssl==0.15.1
+
pyopenssl_trunk: https://github.com/pyca/pyopenssl/archive/master.zip
+
+ docs: Sphinx>=1.4.8
+setenv =
+ # Avoid unnecessary network access when creating virtualenvs for speed.
+ VIRTUALENV_NO_DOWNLOAD=1
+ PIP_DISABLE_PIP_VERSION_CHECK=1
passenv = TERM # ensure colors
commands =
- coverage run --branch --source=treq {envbindir}/trial {posargs:treq}
+ pip list
+ coverage run {envbindir}/trial {posargs:treq}
coverage report -m
[testenv:flake8]
skip_install = True
deps = flake8
-commands = flake8 treq/
+commands = flake8 src/treq/
[testenv:pypi-readme]
deps =
- readme
+ readme_renderer
commands =
python setup.py check -r -s
@@ -42,3 +43,8 @@ deps =
check-manifest
commands =
check-manifest
+
+[testenv:docs]
+changedir = docs
+commands =
+ sphinx-build -b html . html
diff --git a/tox2travis.py b/tox2travis.py
index 942dccd..28dd339 100755
--- a/tox2travis.py
+++ b/tox2travis.py
@@ -1,7 +1,15 @@
#!/usr/bin/env python
+"""
+Generate a Travis CI configuration based on Tox's configured environments.
+Usage:
+
+ tox -l | ./tox2travis.py > .travis.yml
+"""
+
from __future__ import absolute_import, print_function
+import re
import sys
@@ -10,34 +18,54 @@ travis_template = """\
sudo: false
language: python
-python: 2.7
-cache: false
+cache: pip
-env:
- {envs}
+matrix:
+ include:
+ {includes}
+
+ # Don't fail on trunk versions.
+ allow_failures:
+ - env: TOXENV=pypy-twisted_trunk-pyopenssl_trunk
+ - env: TOXENV=py27-twisted_trunk-pyopenssl_trunk
+ - env: TOXENV=py34-twisted_trunk-pyopenssl_trunk
+ - env: TOXENV=py35-twisted_trunk-pyopenssl_trunk
+ - env: TOXENV=py36-twisted_trunk-pyopenssl_trunk
+
+before_install:
+ - |
+ if [[ "${{TOXENV::5}}" == "pypy-" ]]; then
+ PYENV_ROOT="$HOME/.pyenv"
+ git clone --depth 1 https://github.com/yyuu/pyenv.git "$PYENV_ROOT"
+ PATH="$PYENV_ROOT/bin:$PATH"
+ eval "$(pyenv init -)"
+ pyenv install pypy-5.4.1
+ pyenv global pypy-5.4.1
+ fi
+ - pip install --upgrade pip
+ - pip install --upgrade setuptools
install:
- pip install tox codecov
script:
- - tox -e $TOX_ENV
+ - tox
after_success:
- codecov
+after_failure:
+ - |
+ if [[ -f "_trial_temp/httpbin-server-error.log" ]]
+ then
+ echo "httpbin-server-error.log:"
+ cat "_trial_temp/httpbin-server-error.log"
+ fi
+
notifications:
email: false
-# Don't fail on trunk versions.
-matrix:
- allow_failures:
- - env: TOX_ENV=pypy-twisted_trunk-pyopenssl_trunk
- - env: TOX_ENV=py27-twisted_trunk-pyopenssl_trunk
- - env: TOX_ENV=py33-twisted_trunk-pyopenssl_trunk
- - env: TOX_ENV=py34-twisted_trunk-pyopenssl_trunk
- - env: TOX_ENV=py35-twisted_trunk-pyopenssl_trunk
-
branches:
only:
- master
@@ -49,9 +77,25 @@ if __name__ == "__main__":
line = sys.stdin.readline()
tox_envs = []
while line:
- tox_envs.append(line)
+ tox_envs.append(line.strip())
line = sys.stdin.readline()
- print(travis_template.format(
- envs=' '.join(
- '- TOX_ENV={0}'.format(env) for env in tox_envs)))
+ includes = []
+ for tox_env in tox_envs:
+ # Parse the Python version from the tox environment name
+ python_match = re.match(r'^py(?:(\d{2})|py)-', tox_env)
+ if python_match is not None:
+ version = python_match.group(1)
+ if version is not None:
+ python = "'{0}.{1}'".format(version[0], version[1])
+ else:
+ python = 'pypy'
+ else:
+ python = "'2.7'" # Default to Python 2.7 if a version isn't found
+
+ includes.extend([
+ '- python: {0}'.format(python),
+ ' env: TOXENV={0}'.format(tox_env)
+ ])
+
+ print(travis_template.format(includes='\n '.join(includes)))
diff --git a/treq.egg-info/SOURCES.txt b/treq.egg-info/SOURCES.txt
deleted file mode 100644
index 5eca343..0000000
--- a/treq.egg-info/SOURCES.txt
+++ /dev/null
@@ -1,52 +0,0 @@
-LICENSE
-MANIFEST.in
-README.rst
-requirements-dev.txt
-setup.cfg
-setup.py
-tox.ini
-tox2travis.py
-docs/Makefile
-docs/api.rst
-docs/conf.py
-docs/howto.rst
-docs/index.rst
-docs/make.bat
-docs/_static/.keepme
-docs/examples/_utils.py
-docs/examples/basic_auth.py
-docs/examples/basic_get.py
-docs/examples/basic_post.py
-docs/examples/disable_redirects.py
-docs/examples/download_file.py
-docs/examples/query_params.py
-docs/examples/redirects.py
-docs/examples/response_history.py
-docs/examples/using_cookies.py
-treq/__init__.py
-treq/_utils.py
-treq/_version
-treq/api.py
-treq/auth.py
-treq/client.py
-treq/content.py
-treq/multipart.py
-treq/response.py
-treq/testing.py
-treq.egg-info/PKG-INFO
-treq.egg-info/SOURCES.txt
-treq.egg-info/dependency_links.txt
-treq.egg-info/pbr.json
-treq.egg-info/requires.txt
-treq.egg-info/top_level.txt
-treq/test/__init__.py
-treq/test/test_api.py
-treq/test/test_auth.py
-treq/test/test_client.py
-treq/test/test_content.py
-treq/test/test_multipart.py
-treq/test/test_response.py
-treq/test/test_testing.py
-treq/test/test_treq_integration.py
-treq/test/test_utils.py
-treq/test/util.py \ No newline at end of file
diff --git a/treq.egg-info/pbr.json b/treq.egg-info/pbr.json
deleted file mode 100644
index 662c40b..0000000
--- a/treq.egg-info/pbr.json
+++ /dev/null
@@ -1 +0,0 @@
-{"is_release": false, "git_version": "ca78637"} \ No newline at end of file
diff --git a/treq.egg-info/requires.txt b/treq.egg-info/requires.txt
deleted file mode 100644
index 9cb7cc5..0000000
--- a/treq.egg-info/requires.txt
+++ /dev/null
@@ -1,5 +0,0 @@
-requests >= 2.1.0
-service_identity >= 14.0.0
-six
-Twisted >= 14.0.2
-pyOpenSSL >= 0.13
diff --git a/treq/_version b/treq/_version
deleted file mode 100644
index d14dfba..0000000
--- a/treq/_version
+++ /dev/null
@@ -1 +0,0 @@
-15.1.0
diff --git a/treq/response.py b/treq/response.py
deleted file mode 100644
index c71423b..0000000
--- a/treq/response.py
+++ /dev/null
@@ -1,48 +0,0 @@
-from __future__ import absolute_import, division, print_function
-
-from twisted.python.components import proxyForInterface
-from twisted.web.iweb import IResponse
-
-from requests.cookies import cookiejar_from_dict
-
-from treq.content import content, json_content, text_content
-
-
-class _Response(proxyForInterface(IResponse)):
- def __init__(self, original, cookiejar):
- self.original = original
- self._cookiejar = cookiejar
-
- def content(self):
- return content(self.original)
-
- def json(self, *args, **kwargs):
- return json_content(self.original, *args, **kwargs)
-
- def text(self, *args, **kwargs):
- return text_content(self.original, *args, **kwargs)
-
- def history(self):
- if not hasattr(self, "previousResponse"):
- raise NotImplementedError(
- "Twisted < 13.1.0 does not support response history.")
-
- response = self
- history = []
-
- while response.previousResponse is not None:
- history.append(_Response(response.previousResponse,
- self._cookiejar))
- response = response.previousResponse
-
- history.reverse()
- return history
-
- def cookies(self):
- jar = cookiejar_from_dict({})
-
- if self._cookiejar is not None:
- for cookie in self._cookiejar:
- jar.set_cookie(cookie)
-
- return jar
diff --git a/treq/test/test_response.py b/treq/test/test_response.py
deleted file mode 100644
index 9886d8c..0000000
--- a/treq/test/test_response.py
+++ /dev/null
@@ -1,64 +0,0 @@
-from twisted.trial.unittest import TestCase
-
-from twisted import version
-from twisted.python.versions import Version
-
-from twisted.web.http_headers import Headers
-
-from treq.response import _Response
-
-
-skip_history = None
-
-if version < Version("twisted", 13, 1, 0):
- skip_history = "Response history not supported on Twisted < 13.1.0."
-
-
-class FakeResponse(object):
- def __init__(self, code, headers):
- self.code = code
- self.headers = headers
- self.previousResponse = None
-
- def setPreviousResponse(self, response):
- self.previousResponse = response
-
-
-class ResponseTests(TestCase):
- def test_history(self):
- redirect1 = FakeResponse(
- 301,
- Headers({'location': ['http://example.com/']})
- )
-
- redirect2 = FakeResponse(
- 302,
- Headers({'location': ['https://example.com/']})
- )
- redirect2.setPreviousResponse(redirect1)
-
- final = FakeResponse(200, Headers({}))
- final.setPreviousResponse(redirect2)
-
- wrapper = _Response(final, None)
-
- history = wrapper.history()
-
- self.assertEqual(wrapper.code, 200)
- self.assertEqual(history[0].code, 301)
- self.assertEqual(history[1].code, 302)
-
- def test_no_history(self):
- wrapper = _Response(FakeResponse(200, Headers({})), None)
- self.assertEqual(wrapper.history(), [])
-
- if skip_history:
- test_history.skip = skip_history
- test_no_history.skip = skip_history
-
- def test_history_notimplemented(self):
- wrapper = _Response(FakeResponse(200, Headers({})), None)
- self.assertRaises(NotImplementedError, wrapper.history)
-
- if not skip_history:
- test_history_notimplemented.skip = "History supported."
diff --git a/treq/test/util.py b/treq/test/util.py
deleted file mode 100644
index b5fd0e0..0000000
--- a/treq/test/util.py
+++ /dev/null
@@ -1,47 +0,0 @@
-import os
-import platform
-
-import mock
-
-import twisted
-
-from twisted.internet import reactor
-from twisted.internet.task import Clock
-from twisted.trial.unittest import TestCase
-from twisted.python.failure import Failure
-from twisted.python.versions import Version
-
-DEBUG = os.getenv("TREQ_DEBUG", False) == "true"
-
-is_pypy = platform.python_implementation() == 'PyPy'
-
-
-if twisted.version < Version('twisted', 13, 1, 0):
- class TestCase(TestCase):
- def successResultOf(self, d):
- results = []
- d.addBoth(results.append)
-
- if isinstance(results[0], Failure):
- results[0].raiseException()
-
- return results[0]
-
- def failureResultOf(self, d, *errorTypes):
- results = []
- d.addBoth(results.append)
-
- if not isinstance(results[0], Failure):
- self.fail("Expected one of {0} got {1}.".format(
- errorTypes, results[0]))
-
- self.assertTrue(results[0].check(*errorTypes))
- return results[0]
-
-
-def with_clock(fn):
- def wrapper(*args, **kwargs):
- clock = Clock()
- with mock.patch.object(reactor, 'callLater', clock.callLater):
- return fn(*(args + (clock,)), **kwargs)
- return wrapper