Updated version 2.0.3+git20180108 from 'upstream/2.0.3+git20180108'

with Debian dir 701f098ff1fec3409bdda74f3f268f91fb989a9c

15 files changed, 204 insertions, 45 deletions

diff --git a/.atom-build.yml b/.atom-build.yml
new file mode 100644
index 0000000..edfc4c0
--- /dev/null
+++ b/.atom-build.yml
@@ -0,0 +1,14 @@
+cmd: "python"
+name: "build"
+args:
+  - "setup.py"
+  - "sdist"
+targets:
+  test:
+    cmd: "python"
+    name: "test"
+    args:
+      - "setup.py"
+      - "test"
+errorMatch:
+    - ".*File \"(?<file>[\\/0-9a-zA-Z\\._\\-]+)\", line (?<line>\\d+).*"
diff --git a/.gitignore b/.gitignore
index 41df384..324c459 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,11 +1,13 @@
 .buildinfo
 .doctrees
+venv
 
 *.py[co]
 
 # Packages
 *.egg
 *.egg-info
+.eggs
 dist
 build
 eggs
@@ -22,6 +24,7 @@ pip-log.txt
 # Unit test / coverage reports
 .coverage
 .tox
+tests/python_keyring/keyring_pass.cfg
 
 # ofx files in root folder
 /*.ofx
@@ -29,5 +32,8 @@ pip-log.txt
 #Translations
 *.mo
 
+#Vim
+*.swp
+
 #Mr Developer
 .mr.developer.cfg
diff --git a/.travis.yml b/.travis.yml
index 7053b5a..3b1b012 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,11 +1,9 @@
 language: python
 python:
-  - "2.6"
   - "2.7"
-  - "3.2"
   - "3.3"
   - "3.4"
   - "3.5"
 install:
-  - pip install .
+  - pip install -r requirements-test.txt
 script: nosetests --verbose tests/*.py
diff --git a/AUTHORS b/AUTHORS
index 21177c3..68a9bad 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -7,4 +7,10 @@ Jeffrey Paul - Encouraging my to improve this module past homebrew status
 @jbms - Jumpstarting the python 3 support
 Rudd-O - Encouraging security review and communication
 @tgoetze - encoding and windows bug fixes
+lsowen - Additional CLI options
+mattprompt - python 3 fixes
+ridler77 - bug fixes
+gboudreau - TD Bank fix
+fanqiuwen - Discover card fix
+jantman - Additional Discover fix, Vanguard cookie fix
 Everyone who uses this - I'm honored that you find this hobby project useful
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c38def6..ec6b2f7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,14 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [2.0.3] - 2017-04-27
+- Update quicken client version to 2500
+- CLI option to set OFX version
+- CLI option to support changing how far back to download
+- Python 3 bug fixes
+- Fix Discover card header requirements
+- Fix TD Bank header requirements
+
 ## [2.0.2] - 2015-05-30
 - Bug: fix get password on windows
 - Bug: ignore unexpected chars when decoding
diff --git a/ofxclient/account.py b/ofxclient/account.py
index c1ebdaf..686a440 100644
--- a/ofxclient/account.py
+++ b/ofxclient/account.py
@@ -112,9 +112,13 @@ class Account(object):
         :rtype: :py:class:`ofxparser.Ofx`
         """
         if IS_PYTHON_2:
-            return OfxParser.parse(self.download(days=days))
+            return OfxParser.parse(
+                self.download(days=days)
+            )
         else:
-            return OfxParser.parse(BytesIO((((self.download(days=days)).read()).encode())))
+            return OfxParser.parse(
+                BytesIO(self.download(days=days).read().encode())
+            )
 
     def statement(self, days=60):
         """Download the :py:class:`ofxparse.Statement` given the time range
diff --git a/ofxclient/cli.py b/ofxclient/cli.py
index 114ae83..62d8321 100644
--- a/ofxclient/cli.py
+++ b/ofxclient/cli.py
@@ -163,6 +163,8 @@ def view_account_menu(account, args):
         print("  App Ver:               %s" % client.app_version)
         print("  App Id:                %s" % client.app_id)
         print("  OFX Ver:               %s" % client.ofx_version)
+        print("  User-Agent header:     %s" % client.user_agent)
+        print("  Accept header:         %s" % client.accept)
         print("  Config File:           %s" % GlobalConfig.file_name)
 
         menu_item('D', 'Download')
@@ -206,7 +208,7 @@ def login_check_menu(bank_info, args):
             description=bank_info['name'],
             username=username,
             password=password,
-            client_args={ofx_version: args.ofx_version}
+            client_args=client_args_for_bank(bank_info, args.ofx_version)
         )
         try:
             i.authenticate()
@@ -221,6 +223,31 @@ def login_check_menu(bank_info, args):
         return 1
 
 
+def client_args_for_bank(bank_info, ofx_version):
+    """
+    Return the client arguments to use for a particular Institution, as found
+    from ofxhome. This provides us with an extension point to override or
+    augment ofxhome data for specific institutions, such as those that
+    require specific User-Agent headers (or no User-Agent header).
+
+    :param bank_info: OFXHome bank information for the institution, as returned
+      by ``OFXHome.lookup()``
+    :type bank_info: dict
+    :param ofx_version: OFX Version argument specified on command line
+    :type ofx_version: str
+    :return: Client arguments for a specific institution
+    :rtype: dict
+    """
+    client_args = {'ofx_version': str(ofx_version)}
+    if 'ofx.discovercard.com' in bank_info['url']:
+        # Discover needs no User-Agent and no Accept headers
+        client_args['user_agent'] = False
+        client_args['accept'] = False
+    if 'www.accountonline.com' in bank_info['url']:
+        # Citi needs no User-Agent header
+        client_args['user_agent'] = False
+    return client_args
+
 def write_and_handle_download(ofx_data, name):
     outfile = io.open(name, 'w')
     outfile.write(ofx_data.read())
diff --git a/ofxclient/client.py b/ofxclient/client.py
index 5a7dcc0..036dfaf 100644
--- a/ofxclient/client.py
+++ b/ofxclient/client.py
@@ -17,8 +17,10 @@ except ImportError:
 import uuid
 
 DEFAULT_APP_ID = 'QWIN'
-DEFAULT_APP_VERSION = '2200'
+DEFAULT_APP_VERSION = '2500'
 DEFAULT_OFX_VERSION = '102'
+DEFAULT_USER_AGENT = 'httpclient'
+DEFAULT_ACCEPT = '*/*, application/x-ofx'
 
 LINE_ENDING = "\r\n"
 
@@ -40,6 +42,12 @@ class Client:
     :type app_version: string
     :param ofx_version: OFX spec version
     :type ofx_version: string
+    :param user_agent: Value to send for User-Agent HTTP header. Leave as
+      None to send default. Set to False to not send User-Agent header.
+    :type user_agent: str, None or False
+    :param accept: Value to send for Accept HTTP header. Leave as
+      None to send default. Set to False to not send User-Agent header.
+    :type accept: str, None or False
     """
 
     def __init__(
@@ -48,15 +56,39 @@ class Client:
         id=ofx_uid(),
         app_id=DEFAULT_APP_ID,
         app_version=DEFAULT_APP_VERSION,
-        ofx_version=DEFAULT_OFX_VERSION
+        ofx_version=DEFAULT_OFX_VERSION,
+        user_agent=DEFAULT_USER_AGENT,
+        accept=DEFAULT_ACCEPT
     ):
         self.institution = institution
         self.id = id
         self.app_id = app_id
         self.app_version = app_version
         self.ofx_version = ofx_version
+        self.user_agent = user_agent
+        self.accept = accept
+        # used when serializing Institutions
+        self._init_args = {
+            'id': self.id,
+            'app_id': self.app_id,
+            'app_version': self.app_version,
+            'ofx_version': self.ofx_version,
+            'user_agent': self.user_agent,
+            'accept': self.accept
+        }
         self.cookie = 3
 
+    @property
+    def init_args(self):
+        """
+        Return a dict of the arguments used to initialize this client,
+        suitable for use when serializing an Institution.
+
+        :return: constructor arguments
+        :rtype: dict
+        """
+        return self._init_args
+
     def authenticated_query(
         self,
         with_message=None,
@@ -94,26 +126,66 @@ class Client:
         return self.authenticated_query(self._acctreq(date))
 
     def post(self, query):
+        """
+        Wrapper around ``_do_post()`` to handle accounts that require
+        sending back session cookies (``self.set_cookies`` True).
+        """
+        res, response = self._do_post(query)
+        cookies = res.getheader('Set-Cookie', None)
+        if len(response) == 0 and cookies is not None and res.status == 200:
+            logging.debug('Got 0-length 200 response with Set-Cookies header; '
+                          'retrying request with cookies')
+            _, response = self._do_post(query, [('Cookie', cookies)])
+        return response
+
+    def _do_post(self, query, extra_headers=[]):
+        """
+        Do a POST to the Institution.
+
+        :param query: Body content to POST (OFX Query)
+        :type query: str
+        :param extra_headers: Extra headers to send with the request, as a list
+          of (Name, Value) header 2-tuples.
+        :type extra_headers: list
+        :return: 2-tuple of (HTTPResponse, str response body)
+        :rtype: tuple
+        """
         i = self.institution
         logging.debug('posting data to %s' % i.url)
-        logging.debug('---- request ----')
-        logging.debug(query)
         garbage, path = splittype(i.url)
         host, selector = splithost(path)
         h = HTTPSConnection(host, timeout=60)
-        h.request('POST', selector, query,
-                  {
-                      "Content-type": "application/x-ofx",
-                      "Accept": "*/*, application/x-ofx"
-                  })
+        # Discover requires a particular ordering of headers, so send the
+        # request step by step.
+        h.putrequest('POST', selector, skip_host=True,
+                     skip_accept_encoding=True)
+        headers = [
+            ('Content-Type', 'application/x-ofx'),
+            ('Host', host),
+            ('Content-Length', len(query)),
+            ('Connection', 'Keep-Alive')
+        ]
+        if self.accept:
+            headers.append(('Accept', self.accept))
+        if self.user_agent:
+            headers.append(('User-Agent', self.user_agent))
+        for ehname, ehval in extra_headers:
+            headers.append((ehname, ehval))
+        logging.debug('---- request headers ----')
+        for hname, hval in headers:
+            logging.debug('%s: %s', hname, hval)
+            h.putheader(hname, hval)
+        logging.debug('---- request body (query) ----')
+        logging.debug(query)
+        h.endheaders(query.encode())
         res = h.getresponse()
         response = res.read().decode('ascii', 'ignore')
         logging.debug('---- response ----')
         logging.debug(res.__dict__)
+        logging.debug('Headers: %s', res.getheaders())
         logging.debug(response)
         res.close()
-
-        return response
+        return res, response
 
     def next_cookie(self):
         self.cookie += 1
diff --git a/ofxclient/config.py b/ofxclient/config.py
index 0dfeaac..7a6757c 100644
--- a/ofxclient/config.py
+++ b/ofxclient/config.py
@@ -100,6 +100,8 @@ class SecurableConfigParser(ConfigParser):
         for k, v in ConfigParser.items(self, section):
             if self.is_secure_option(section, k):
                 v = self.get(section, k)
+            if v == '!!False!!':
+                v = False
             items.append((k, v))
         return items
 
@@ -116,6 +118,8 @@ class SecurableConfigParser(ConfigParser):
     def set(self, section, option, value):
         """Set an option value. Knows how to set options properly marked
         as secure."""
+        if not value:
+            value = '!!False!!'
         if self.is_secure_option(section, option):
             self.set_secure(section, option, value)
         else:
@@ -139,10 +143,14 @@ class SecurableConfigParser(ConfigParser):
         if self.is_secure_option(section, option) and self.keyring_available:
             s_option = "%s%s" % (section, option)
             if self._unsaved.get(s_option, [''])[0] == 'set':
-                return self._unsaved[s_option][1]
+                res = self._unsaved[s_option][1]
             else:
-                return keyring.get_password(self.keyring_name, s_option)
-        return ConfigParser.get(self, section, option, *args)
+                res = keyring.get_password(self.keyring_name, s_option)
+        else:
+            res = ConfigParser.get(self, section, option, *args)
+        if res == '!!False!!':
+            return False
+        return res
 
     def remove_option(self, section, option):
         """Removes the option from ConfigParser as well as
diff --git a/ofxclient/institution.py b/ofxclient/institution.py
index 7e38096..3a06588 100644
--- a/ofxclient/institution.py
+++ b/ofxclient/institution.py
@@ -8,7 +8,7 @@ try:
 except ImportError:
     # python 2
     from StringIO import StringIO
-    IS_PYTHON_2 = Trues
+    IS_PYTHON_2 = True
 
 from bs4 import BeautifulSoup
 from ofxparse import OfxParser
@@ -145,7 +145,7 @@ class Institution(object):
         if IS_PYTHON_2:
             parsed = OfxParser.parse(resp_handle)
         else:
-            parsed = OfxParser.parse(BytesIO((((resp_handle).read()).encode())))
+            parsed = OfxParser.parse(BytesIO(resp_handle.read().encode()))
 
         return [Account.from_ofxparse(a, institution=self)
                 for a in parsed.accounts]
@@ -165,22 +165,14 @@ class Institution(object):
             'password':    'Customer password',
             'description': 'descr',
             'client_args': {
-                'id':          'random client id - see Client() for default',
-                'app_id':      'app name - see Client() for default',
-                'app_version': 'app version - see Client() for default',
-                'ofx_version': 'ofx version - see Client() for default',
+                'id':     'random client id - see Client() for default',
+                'app_id': 'app name - see Client() for default',
+                '...':    'see Client() for other options'
             }
           }
 
         :rtype: nested dictionary
         """
-        client = self.client()
-        client_args = {
-            'id': client.id,
-            'app_id': client.app_id,
-            'app_version': client.app_version,
-            'ofx_version': client.ofx_version,
-        }
         return {
             'id': self.id,
             'org': self.org,
@@ -189,7 +181,7 @@ class Institution(object):
             'username': self.username,
             'password': self.password,
             'description': self.description,
-            'client_args': client_args,
+            'client_args': self.client().init_args,
             'local_id': self.local_id()
         }
 
diff --git a/ofxclient/version.py b/ofxclient/version.py
index 18e8949..492f5aa 100644
--- a/ofxclient/version.py
+++ b/ofxclient/version.py
@@ -1,2 +1,2 @@
 from __future__ import unicode_literals
-__version__ = '2.0.2'
+__version__ = '2.0.3'
diff --git a/requirements-test.txt b/requirements-test.txt
new file mode 100644
index 0000000..8bb2683
--- /dev/null
+++ b/requirements-test.txt
@@ -0,0 +1,3 @@
+-r requirements.txt
+mock
+nose
diff --git a/requirements.txt b/requirements.txt
index 5aea447..3ac7954 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,8 +1,8 @@
-argparse ==1.4.1; python_version < '2.7'
-beautifulsoup4 ==4.4.1
-keyring ==8.4.1
-ofxhome ==0.3.3
-ofxparse ==0.14
-lxml ==3.5.0
-keyrings.alt ==1.1.1
-pycrypto ==2.6.1
+argparse==1.4.1; python_version < '2.7'
+beautifulsoup4==4.4.1
+keyring==8.4.1
+ofxhome==0.3.3
+ofxparse==0.14
+lxml>=3.5.0
+keyrings.alt==1.1.1
+pycrypto==2.6.1
diff --git a/setup.py b/setup.py
index 696ab5f..3b77751 100644
--- a/setup.py
+++ b/setup.py
@@ -1,5 +1,4 @@
 from setuptools import setup, find_packages
-import os
 import re
 
 VERSIONFILE = "ofxclient/version.py"
diff --git a/tests/ofxconfig.py b/tests/ofxconfig.py
index fdbb251..8800146 100644
--- a/tests/ofxconfig.py
+++ b/tests/ofxconfig.py
@@ -1,7 +1,13 @@
+import keyring
+from keyrings.alt.file import PlaintextKeyring
 import os
 import os.path
 import tempfile
 import unittest
+try:
+    from test.support import EnvironmentVarGuard
+except ImportError:
+    from test.test_support import EnvironmentVarGuard
 
 import ofxclient.config
 from ofxclient.config import OfxConfig
@@ -11,8 +17,15 @@ from ofxclient import Institution, CreditCardAccount
 class OfxConfigTests(unittest.TestCase):
 
     def setUp(self):
+        keyring.set_keyring(PlaintextKeyring())
+
+        self.env = EnvironmentVarGuard()
         self.temp_file = tempfile.NamedTemporaryFile()
 
+        test_path = os.path.dirname(os.path.realpath(__file__))
+        self.env['XDG_DATA_HOME'] = test_path
+        self.env['XDG_CONFIG_HOME'] = test_path
+
     def tearDown(self):
         self.temp_file.close()
 
@@ -22,7 +35,7 @@ class OfxConfigTests(unittest.TestCase):
 
         self.assertFalse(os.path.exists(file_name))
 
-        c = OfxConfig(file_name=file_name)
+        c = OfxConfig(file_name=file_name)  # noqa
         self.assertTrue(os.path.exists(file_name))
 
         os.remove(file_name)
@@ -72,6 +85,9 @@ class OfxConfigTests(unittest.TestCase):
         if not ofxclient.config.KEYRING_AVAILABLE:
             return
 
+        # always skip these for now
+        return
+
         c = OfxConfig(file_name=self.temp_file.name)
 
         i = Institution(
@@ -95,6 +111,9 @@ class OfxConfigTests(unittest.TestCase):
         if not ofxclient.config.KEYRING_AVAILABLE:
             return
 
+        # always skip these for now
+        return
+
         c = OfxConfig(file_name=self.temp_file.name)
         i = Institution(
                 id='1',
@@ -123,6 +142,9 @@ class OfxConfigTests(unittest.TestCase):
         if not ofxclient.config.KEYRING_AVAILABLE:
             return
 
+        # always skip these for now
+        return
+
         c = OfxConfig(file_name=self.temp_file.name)
         i = Institution(
                 id='1',