diff options
| -rw-r--r-- | .gitignore | 4 | ||||
| -rw-r--r-- | setup.py | 4 | ||||
| -rw-r--r-- | src/etcd/__init__.py | 20 | ||||
| -rw-r--r-- | src/etcd/client.py | 69 | ||||
| -rw-r--r-- | src/etcd/tests/integration/test_simple.py | 1 | ||||
| -rw-r--r-- | src/etcd/tests/unit/test_result.py | 144 |
6 files changed, 205 insertions, 37 deletions
@@ -4,9 +4,13 @@ bin
develop-eggs
eggs
+.eggs
+.idea
*.egg-info
tmp
build
dist
+docs
+etcd
.coverage
@@ -18,7 +18,8 @@ test_requires = [ 'pyOpenSSL>=0.14' ] -setup(name='python-etcd', +setup( + name='python-etcd', version=version, description="A python client for etcd", long_description=README + '\n\n' + NEWS, @@ -42,5 +43,4 @@ setup(name='python-etcd', install_requires=install_requires, tests_require=test_requires, test_suite='nose.collector', - ) diff --git a/src/etcd/__init__.py b/src/etcd/__init__.py index 2a5992b..b532be6 100644 --- a/src/etcd/__init__.py +++ b/src/etcd/__init__.py @@ -76,13 +76,14 @@ class EtcdResult(object): #if the current result is a leaf, return itself yield self return - for n in self._children: - node = EtcdResult(None, n) + else: + # node is not a leaf if not leaves_only: - #Return also dirs, not just value nodes - yield node - for child in node.get_subtree(leaves_only=leaves_only): - yield child + yield self + for n in self._children: + node = EtcdResult(None, n) + for child in node.get_subtree(leaves_only=leaves_only): + yield child return @property @@ -120,7 +121,7 @@ class EtcdException(Exception): Generic Etcd Exception. """ def __init__(self, message=None, payload=None): - super(Exception, self).__init__(message) + super(EtcdException, self).__init__(message) self.payload = payload @@ -193,7 +194,10 @@ class EtcdConnectionFailed(EtcdException): """ Connection to etcd failed. """ - pass + def __init__(self, message=None, payload=None, cause=None): + super(EtcdConnectionFailed, self).__init__(message=message, + payload=payload) + self.cause = cause class EtcdWatcherCleared(EtcdException): diff --git a/src/etcd/client.py b/src/etcd/client.py index fe766a5..03b0451 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -688,8 +688,13 @@ class Client(object): def _result_from_response(self, response): """ Creates an EtcdResult from json dictionary """ + raw_response = response.data + try: + res = json.loads(raw_response.decode('utf-8')) + except (TypeError, ValueError, UnicodeError) as e: + raise etcd.EtcdException( + 'Server response was not valid JSON: %r' % e) try: - res = json.loads(response.data.decode('utf-8')) r = etcd.EtcdResult(**res) if response.status == 201: r.newKey = True @@ -697,9 +702,9 @@ class Client(object): return r except Exception as e: raise etcd.EtcdException( - 'Unable to decode server response: %s' % e) + 'Unable to decode server response: %r' % e) - def _next_server(self): + def _next_server(self, cause=None): """ Selects the next server in the list, refreshes the server list. """ _log.debug("Selection next machine in cache. Available machines: %s", self._machines_cache) @@ -707,7 +712,8 @@ class Client(object): mach = self._machines_cache.pop() except IndexError: _log.error("Machines cache is empty, no machines to try.") - raise etcd.EtcdConnectionFailed('No more machines in the cluster') + raise etcd.EtcdConnectionFailed('No more machines in the cluster', + cause=cause) else: _log.info("Selected new etcd server %s", mach) return mach @@ -752,7 +758,15 @@ class Client(object): else: raise etcd.EtcdException( 'HTTP method {} not supported'.format(method)) - + + # Check the cluster ID hasn't changed under us. We use + # preload_content=False above so we can read the headers + # before we wait for the content of a watch. + self._check_cluster_id(response) + # Now force the data to be preloaded in order to trigger any + # IO-related errors in this method rather than when we try to + # access it later. + _ = response.data # urllib3 doesn't wrap all httplib exceptions and earlier versions # don't wrap socket errors either. except (urllib3.exceptions.HTTPError, @@ -765,35 +779,18 @@ class Client(object): "server.") # _next_server() raises EtcdException if there are no # machines left to try, breaking out of the loop. - self._base_uri = self._next_server() + self._base_uri = self._next_server(cause=e) some_request_failed = True else: _log.debug("Reconnection disabled, giving up.") raise etcd.EtcdConnectionFailed( - "Connection to etcd failed due to %r" % e) + "Connection to etcd failed due to %r" % e, + cause=e + ) except: _log.exception("Unexpected request failure, re-raising.") raise - - else: - # Check the cluster ID hasn't changed under us. We use - # preload_content=False above so we can read the headers - # before we wait for the content of a long poll. - cluster_id = response.getheader("x-etcd-cluster-id") - id_changed = (self.expected_cluster_id - and cluster_id is not None and - cluster_id != self.expected_cluster_id) - # Update the ID so we only raise the exception once. - old_expected_cluster_id = self.expected_cluster_id - self.expected_cluster_id = cluster_id - if id_changed: - # Defensive: clear the pool so that we connect afresh next - # time. - self.http.clear() - raise etcd.EtcdClusterIdChanged( - 'The UUID of the cluster changed from {} to ' - '{}.'.format(old_expected_cluster_id, cluster_id)) - + if some_request_failed: if not self._use_proxies: # The cluster may have changed since last invocation @@ -801,6 +798,24 @@ class Client(object): self._machines_cache.remove(self._base_uri) return self._handle_server_response(response) + def _check_cluster_id(self, response): + cluster_id = response.getheader("x-etcd-cluster-id") + if not cluster_id: + _log.warning("etcd response did not contain a cluster ID") + return + id_changed = (self.expected_cluster_id and + cluster_id != self.expected_cluster_id) + # Update the ID so we only raise the exception once. + old_expected_cluster_id = self.expected_cluster_id + self.expected_cluster_id = cluster_id + if id_changed: + # Defensive: clear the pool so that we connect afresh next + # time. + self.http.clear() + raise etcd.EtcdClusterIdChanged( + 'The UUID of the cluster changed from {} to ' + '{}.'.format(old_expected_cluster_id, cluster_id)) + def _handle_server_response(self, response): """ Handles the server response """ if response.status in [200, 201]: diff --git a/src/etcd/tests/integration/test_simple.py b/src/etcd/tests/integration/test_simple.py index c275d6e..da0954d 100644 --- a/src/etcd/tests/integration/test_simple.py +++ b/src/etcd/tests/integration/test_simple.py @@ -64,6 +64,7 @@ class TestSimple(EtcdIntegrationTest): def test_machines(self): """ INTEGRATION: retrieve machines """ self.assertEquals(self.client.machines[0], 'http://127.0.0.1:6001') + def test_leader(self): """ INTEGRATION: retrieve leader """ self.assertEquals(self.client.leader['clientURLs'], ['http://127.0.0.1:6001']) diff --git a/src/etcd/tests/unit/test_result.py b/src/etcd/tests/unit/test_result.py new file mode 100644 index 0000000..cb1414b --- /dev/null +++ b/src/etcd/tests/unit/test_result.py @@ -0,0 +1,144 @@ +import etcd +import unittest +import json +import urllib3 + +try: + import mock +except ImportError: + from unittest import mock + +class TestEtcdResult(unittest.TestCase): + + def test_get_subtree_1_level(self): + """ + Test get_subtree() for a read with tree 1 level deep. + """ + response = {"node": { + 'key': "/test", + 'value': "hello", + 'expiration': None, + 'ttl': None, + 'modifiedIndex': 5, + 'createdIndex': 1, + 'newKey': False, + 'dir': False, + }} + result = etcd.EtcdResult(**response) + self.assertEqual(result.key, response["node"]["key"]) + self.assertEqual(result.value, response["node"]["value"]) + + # Get subtree returns itself, whether or not leaves_only + subtree = list(result.get_subtree(leaves_only=True)) + self.assertListEqual([result], subtree) + subtree = list(result.get_subtree(leaves_only=False)) + self.assertListEqual([result], subtree) + + def test_get_subtree_2_level(self): + """ + Test get_subtree() for a read with tree 2 levels deep. + """ + leaf0 = { + 'key': "/test/leaf0", + 'value': "hello1", + 'expiration': None, + 'ttl': None, + 'modifiedIndex': 5, + 'createdIndex': 1, + 'newKey': False, + 'dir': False, + } + leaf1 = { + 'key': "/test/leaf1", + 'value': "hello2", + 'expiration': None, + 'ttl': None, + 'modifiedIndex': 6, + 'createdIndex': 2, + 'newKey': False, + 'dir': False, + } + testnode = {"node": { + 'key': "/test/", + 'expiration': None, + 'ttl': None, + 'modifiedIndex': 6, + 'createdIndex': 2, + 'newKey': False, + 'dir': True, + 'nodes': [leaf0, leaf1] + }} + result = etcd.EtcdResult(**testnode) + self.assertEqual(result.key, "/test/") + self.assertTrue(result.dir) + + # Get subtree returns just two leaves for leaves only. + subtree = list(result.get_subtree(leaves_only=True)) + self.assertEqual(subtree[0].key, "/test/leaf0") + self.assertEqual(subtree[1].key, "/test/leaf1") + self.assertEqual(len(subtree), 2) + + # Get subtree returns leaves and directory. + subtree = list(result.get_subtree(leaves_only=False)) + self.assertEqual(subtree[0].key, "/test/") + self.assertEqual(subtree[1].key, "/test/leaf0") + self.assertEqual(subtree[2].key, "/test/leaf1") + self.assertEqual(len(subtree), 3) + + def test_get_subtree_3_level(self): + """ + Test get_subtree() for a read with tree 3 levels deep. + """ + leaf0 = { + 'key': "/test/mid0/leaf0", + 'value': "hello1", + } + leaf1 = { + 'key': "/test/mid0/leaf1", + 'value': "hello2", + } + leaf2 = { + 'key': "/test/mid1/leaf2", + 'value': "hello1", + } + leaf3 = { + 'key': "/test/mid1/leaf3", + 'value': "hello2", + } + mid0 = { + 'key': "/test/mid0/", + 'dir': True, + 'nodes': [leaf0, leaf1] + } + mid1 = { + 'key': "/test/mid1/", + 'dir': True, + 'nodes': [leaf2, leaf3] + } + testnode = {"node": { + 'key': "/test/", + 'dir': True, + 'nodes': [mid0, mid1] + }} + result = etcd.EtcdResult(**testnode) + self.assertEqual(result.key, "/test/") + self.assertTrue(result.dir) + + # Get subtree returns just two leaves for leaves only. + subtree = list(result.get_subtree(leaves_only=True)) + self.assertEqual(subtree[0].key, "/test/mid0/leaf0") + self.assertEqual(subtree[1].key, "/test/mid0/leaf1") + self.assertEqual(subtree[2].key, "/test/mid1/leaf2") + self.assertEqual(subtree[3].key, "/test/mid1/leaf3") + self.assertEqual(len(subtree), 4) + + # Get subtree returns leaves and directory. + subtree = list(result.get_subtree(leaves_only=False)) + self.assertEqual(subtree[0].key, "/test/") + self.assertEqual(subtree[1].key, "/test/mid0/") + self.assertEqual(subtree[2].key, "/test/mid0/leaf0") + self.assertEqual(subtree[3].key, "/test/mid0/leaf1") + self.assertEqual(subtree[4].key, "/test/mid1/") + self.assertEqual(subtree[5].key, "/test/mid1/leaf2") + self.assertEqual(subtree[6].key, "/test/mid1/leaf3") + self.assertEqual(len(subtree), 7) |