From fe684098037ca09db0c8e3dfb989ee400fddbd24 Mon Sep 17 00:00:00 2001 From: Giuseppe Lavagetto Date: Sat, 20 Jun 2015 13:44:52 +0200 Subject: Add srv record-based DNS discovery. We use the same keys used by confd (https://github.com/kelseyhightower/confd) to allow service discovery via DNS. --- .travis.yml | 4 ++-- README.rst | 2 ++ buildout.cfg | 10 ++++++++++ setup.py | 9 ++++++++- src/etcd/client.py | 29 +++++++++++++++++++++++++++-- src/etcd/tests/unit/test_client.py | 25 +++++++++++++++++++++++++ 6 files changed, 74 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 46a2576..1538059 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,10 @@ language: python python: - "2.7" - - "3.3" + - "3.5" before_install: - - ./build_etcd.sh v2.0.10 + - ./build_etcd.sh v2.2.0 - pip install --upgrade setuptools # command to install dependencies diff --git a/README.rst b/README.rst index 163d257..ee5b70c 100644 --- a/README.rst +++ b/README.rst @@ -41,6 +41,8 @@ Create a client object client = etcd.Client(port=4002) client = etcd.Client(host='127.0.0.1', port=4003) client = etcd.Client(host='127.0.0.1', port=4003, allow_redirect=False) # wont let you run sensitive commands on non-leader machines, default is true + # If you have defined a SRV record for _etcd._tcp.example.com pointing to the clients + client = etcd.Client(srv_domain='example.com', protocol="https") # create a client against https://api.example.com:443/etcd client = etcd.Client(host='api.example.com', protocol='https', port=443, version_prefix='/etcd') Write a key diff --git a/buildout.cfg b/buildout.cfg index bd498e5..cba64c5 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -6,6 +6,7 @@ develop = . eggs = urllib3==1.7.1 pyOpenSSL==0.13.1 + ${deps:extraeggs} [python] recipe = zc.recipe.egg @@ -21,3 +22,12 @@ eggs = ${python:eggs} recipe = collective.recipe.sphinxbuilder source = ${buildout:directory}/docs-source build = ${buildout:directory}/docs + + +[deps:python2] +extraeggs = + dnspython==1.12.0 + +[deps:python3] +extraeggs = + dnspython3==1.12.0 diff --git a/setup.py b/setup.py index b496fe2..3d0d450 100644 --- a/setup.py +++ b/setup.py @@ -8,8 +8,15 @@ NEWS = open(os.path.join(here, 'NEWS.txt')).read() version = '0.4.2' +# Dnspython is two different packages depending on python version +if sys.version_info.major == 2: + dns = 'dnspython' +else: + dns = 'dnspython3' + install_requires = [ - 'urllib3>=1.7.1' + 'urllib3>=1.7.1', + dns ] test_requires = [ diff --git a/src/etcd/client.py b/src/etcd/client.py index 7d32ccf..c0cae84 100644 --- a/src/etcd/client.py +++ b/src/etcd/client.py @@ -18,6 +18,7 @@ import urllib3 import urllib3.util import json import ssl +import dns.resolver import etcd try: @@ -46,6 +47,7 @@ class Client(object): self, host='127.0.0.1', port=4001, + srv_domain=None, version_prefix='/v2', read_timeout=60, allow_redirect=True, @@ -67,6 +69,8 @@ class Client(object): port (int): Port used to connect to etcd. + srv_domain (str): Domain to search the SRV record for cluster autodiscovery. + version_prefix (str): Url or version prefix in etcd url (default=/v2). read_timeout (int): max seconds to wait for a read. @@ -98,8 +102,15 @@ class Client(object): by host. By default this will use up to 10 connections. """ - _log.debug("New etcd client created for %s:%s%s", - host, port, version_prefix) + + # If a DNS record is provided, use it to get the hosts list + if srv_domain is not None: + try: + host = self._discover(srv_domain) + except Exception as e: + _log.error("Could not discover the etcd hosts from %s: %s", + srv_domain, e) + self._protocol = protocol def uri(protocol, host, port): @@ -153,6 +164,8 @@ class Client(object): self.http = urllib3.PoolManager(num_pools=10, **kw) + _log.debug("New etcd client created for %s", self.base_uri) + if self._allow_reconnect: # we need the set of servers in the cluster in order to try # reconnecting upon error. The cluster members will be @@ -174,6 +187,18 @@ class Client(object): _log.debug("Machines cache initialised to %s", self._machines_cache) + def _discover(self, domain): + srv_name = "_etcd._tcp.{}".format(domain) + answers = dns.resolver.query(srv_name, 'SRV') + hosts = [] + for answer in answers: + hosts.append( + (answer.target.to_text(omit_final_dot=True), answer.port)) + _log.debug("Found %s", hosts) + if not len(hosts): + raise ValueError("The SRV record is present but no host were found") + return tuple(hosts) + @property def base_uri(self): """URI used by the client to connect to etcd.""" diff --git a/src/etcd/tests/unit/test_client.py b/src/etcd/tests/unit/test_client.py index 2e09d7c..e5d1099 100644 --- a/src/etcd/tests/unit/test_client.py +++ b/src/etcd/tests/unit/test_client.py @@ -1,5 +1,12 @@ import unittest import etcd +import dns.name +import dns.rdtypes.IN.SRV +import dns.resolver +try: + import mock +except ImportError: + from unittest import mock class TestClient(unittest.TestCase): @@ -97,3 +104,21 @@ class TestClient(unittest.TestCase): allow_reconnect=True, use_proxies=True, ) + + def test_discover(self): + """Tests discovery.""" + answers = [] + for i in range(1,3): + r = mock.create_autospec(dns.rdtypes.IN.SRV.SRV) + r.port = 2379 + r.target = dns.name.from_unicode(u'etcd{}.example.com'.format(i)) + answers.append(r) + dns.resolver.query = mock.create_autospec(dns.resolver.query, return_value=answers) + self.machines = etcd.Client.machines + etcd.Client.machines = mock.create_autospec(etcd.Client.machines, return_value=[u'https://etcd2.example.com:2379']) + c = etcd.Client(srv_domain="example.com", allow_reconnect=True, protocol="https") + etcd.Client.machines = self.machines + self.assertEquals(c.host, u'etcd1.example.com') + self.assertEquals(c.port, 2379) + self.assertEquals(c._machines_cache, + [u'https://etcd2.example.com:2379']) -- cgit v1.2.3