From 48e04477c0bc81a882422828ac1d09012a20da6a Mon Sep 17 00:00:00 2001 From: Andrew Shadura Date: Mon, 30 Apr 2018 10:52:49 +0200 Subject: New upstream version 2.0.3+git20180108 --- .atom-build.yml | 14 ++++++++ .gitignore | 6 ++++ .travis.yml | 4 +-- AUTHORS | 6 ++++ CHANGELOG.md | 8 +++++ ofxclient/account.py | 8 +++-- ofxclient/cli.py | 29 ++++++++++++++- ofxclient/client.py | 94 ++++++++++++++++++++++++++++++++++++++++++------ ofxclient/config.py | 14 ++++++-- ofxclient/institution.py | 20 ++++------- ofxclient/version.py | 2 +- requirements-test.txt | 3 ++ requirements.txt | 16 ++++----- setup.py | 1 - tests/ofxconfig.py | 24 ++++++++++++- 15 files changed, 204 insertions(+), 45 deletions(-) create mode 100644 .atom-build.yml create mode 100644 requirements-test.txt diff --git a/.atom-build.yml b/.atom-build.yml new file mode 100644 index 0000000..edfc4c0 --- /dev/null +++ b/.atom-build.yml @@ -0,0 +1,14 @@ +cmd: "python" +name: "build" +args: + - "setup.py" + - "sdist" +targets: + test: + cmd: "python" + name: "test" + args: + - "setup.py" + - "test" +errorMatch: + - ".*File \"(?[\\/0-9a-zA-Z\\._\\-]+)\", line (?\\d+).*" diff --git a/.gitignore b/.gitignore index 41df384..324c459 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,13 @@ .buildinfo .doctrees +venv *.py[co] # Packages *.egg *.egg-info +.eggs dist build eggs @@ -22,6 +24,7 @@ pip-log.txt # Unit test / coverage reports .coverage .tox +tests/python_keyring/keyring_pass.cfg # ofx files in root folder /*.ofx @@ -29,5 +32,8 @@ pip-log.txt #Translations *.mo +#Vim +*.swp + #Mr Developer .mr.developer.cfg diff --git a/.travis.yml b/.travis.yml index 7053b5a..3b1b012 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,9 @@ language: python python: - - "2.6" - "2.7" - - "3.2" - "3.3" - "3.4" - "3.5" install: - - pip install . + - pip install -r requirements-test.txt script: nosetests --verbose tests/*.py diff --git a/AUTHORS b/AUTHORS index 21177c3..68a9bad 100644 --- a/AUTHORS +++ b/AUTHORS @@ -7,4 +7,10 @@ Jeffrey Paul - Encouraging my to improve this module past homebrew status @jbms - Jumpstarting the python 3 support Rudd-O - Encouraging security review and communication @tgoetze - encoding and windows bug fixes +lsowen - Additional CLI options +mattprompt - python 3 fixes +ridler77 - bug fixes +gboudreau - TD Bank fix +fanqiuwen - Discover card fix +jantman - Additional Discover fix, Vanguard cookie fix Everyone who uses this - I'm honored that you find this hobby project useful diff --git a/CHANGELOG.md b/CHANGELOG.md index c38def6..ec6b2f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,14 @@ # Change Log All notable changes to this project will be documented in this file. +## [2.0.3] - 2017-04-27 +- Update quicken client version to 2500 +- CLI option to set OFX version +- CLI option to support changing how far back to download +- Python 3 bug fixes +- Fix Discover card header requirements +- Fix TD Bank header requirements + ## [2.0.2] - 2015-05-30 - Bug: fix get password on windows - Bug: ignore unexpected chars when decoding diff --git a/ofxclient/account.py b/ofxclient/account.py index c1ebdaf..686a440 100644 --- a/ofxclient/account.py +++ b/ofxclient/account.py @@ -112,9 +112,13 @@ class Account(object): :rtype: :py:class:`ofxparser.Ofx` """ if IS_PYTHON_2: - return OfxParser.parse(self.download(days=days)) + return OfxParser.parse( + self.download(days=days) + ) else: - return OfxParser.parse(BytesIO((((self.download(days=days)).read()).encode()))) + return OfxParser.parse( + BytesIO(self.download(days=days).read().encode()) + ) def statement(self, days=60): """Download the :py:class:`ofxparse.Statement` given the time range diff --git a/ofxclient/cli.py b/ofxclient/cli.py index 114ae83..62d8321 100644 --- a/ofxclient/cli.py +++ b/ofxclient/cli.py @@ -163,6 +163,8 @@ def view_account_menu(account, args): print(" App Ver: %s" % client.app_version) print(" App Id: %s" % client.app_id) print(" OFX Ver: %s" % client.ofx_version) + print(" User-Agent header: %s" % client.user_agent) + print(" Accept header: %s" % client.accept) print(" Config File: %s" % GlobalConfig.file_name) menu_item('D', 'Download') @@ -206,7 +208,7 @@ def login_check_menu(bank_info, args): description=bank_info['name'], username=username, password=password, - client_args={ofx_version: args.ofx_version} + client_args=client_args_for_bank(bank_info, args.ofx_version) ) try: i.authenticate() @@ -221,6 +223,31 @@ def login_check_menu(bank_info, args): return 1 +def client_args_for_bank(bank_info, ofx_version): + """ + Return the client arguments to use for a particular Institution, as found + from ofxhome. This provides us with an extension point to override or + augment ofxhome data for specific institutions, such as those that + require specific User-Agent headers (or no User-Agent header). + + :param bank_info: OFXHome bank information for the institution, as returned + by ``OFXHome.lookup()`` + :type bank_info: dict + :param ofx_version: OFX Version argument specified on command line + :type ofx_version: str + :return: Client arguments for a specific institution + :rtype: dict + """ + client_args = {'ofx_version': str(ofx_version)} + if 'ofx.discovercard.com' in bank_info['url']: + # Discover needs no User-Agent and no Accept headers + client_args['user_agent'] = False + client_args['accept'] = False + if 'www.accountonline.com' in bank_info['url']: + # Citi needs no User-Agent header + client_args['user_agent'] = False + return client_args + def write_and_handle_download(ofx_data, name): outfile = io.open(name, 'w') outfile.write(ofx_data.read()) diff --git a/ofxclient/client.py b/ofxclient/client.py index 5a7dcc0..036dfaf 100644 --- a/ofxclient/client.py +++ b/ofxclient/client.py @@ -17,8 +17,10 @@ except ImportError: import uuid DEFAULT_APP_ID = 'QWIN' -DEFAULT_APP_VERSION = '2200' +DEFAULT_APP_VERSION = '2500' DEFAULT_OFX_VERSION = '102' +DEFAULT_USER_AGENT = 'httpclient' +DEFAULT_ACCEPT = '*/*, application/x-ofx' LINE_ENDING = "\r\n" @@ -40,6 +42,12 @@ class Client: :type app_version: string :param ofx_version: OFX spec version :type ofx_version: string + :param user_agent: Value to send for User-Agent HTTP header. Leave as + None to send default. Set to False to not send User-Agent header. + :type user_agent: str, None or False + :param accept: Value to send for Accept HTTP header. Leave as + None to send default. Set to False to not send User-Agent header. + :type accept: str, None or False """ def __init__( @@ -48,15 +56,39 @@ class Client: id=ofx_uid(), app_id=DEFAULT_APP_ID, app_version=DEFAULT_APP_VERSION, - ofx_version=DEFAULT_OFX_VERSION + ofx_version=DEFAULT_OFX_VERSION, + user_agent=DEFAULT_USER_AGENT, + accept=DEFAULT_ACCEPT ): self.institution = institution self.id = id self.app_id = app_id self.app_version = app_version self.ofx_version = ofx_version + self.user_agent = user_agent + self.accept = accept + # used when serializing Institutions + self._init_args = { + 'id': self.id, + 'app_id': self.app_id, + 'app_version': self.app_version, + 'ofx_version': self.ofx_version, + 'user_agent': self.user_agent, + 'accept': self.accept + } self.cookie = 3 + @property + def init_args(self): + """ + Return a dict of the arguments used to initialize this client, + suitable for use when serializing an Institution. + + :return: constructor arguments + :rtype: dict + """ + return self._init_args + def authenticated_query( self, with_message=None, @@ -94,26 +126,66 @@ class Client: return self.authenticated_query(self._acctreq(date)) def post(self, query): + """ + Wrapper around ``_do_post()`` to handle accounts that require + sending back session cookies (``self.set_cookies`` True). + """ + res, response = self._do_post(query) + cookies = res.getheader('Set-Cookie', None) + if len(response) == 0 and cookies is not None and res.status == 200: + logging.debug('Got 0-length 200 response with Set-Cookies header; ' + 'retrying request with cookies') + _, response = self._do_post(query, [('Cookie', cookies)]) + return response + + def _do_post(self, query, extra_headers=[]): + """ + Do a POST to the Institution. + + :param query: Body content to POST (OFX Query) + :type query: str + :param extra_headers: Extra headers to send with the request, as a list + of (Name, Value) header 2-tuples. + :type extra_headers: list + :return: 2-tuple of (HTTPResponse, str response body) + :rtype: tuple + """ i = self.institution logging.debug('posting data to %s' % i.url) - logging.debug('---- request ----') - logging.debug(query) garbage, path = splittype(i.url) host, selector = splithost(path) h = HTTPSConnection(host, timeout=60) - h.request('POST', selector, query, - { - "Content-type": "application/x-ofx", - "Accept": "*/*, application/x-ofx" - }) + # Discover requires a particular ordering of headers, so send the + # request step by step. + h.putrequest('POST', selector, skip_host=True, + skip_accept_encoding=True) + headers = [ + ('Content-Type', 'application/x-ofx'), + ('Host', host), + ('Content-Length', len(query)), + ('Connection', 'Keep-Alive') + ] + if self.accept: + headers.append(('Accept', self.accept)) + if self.user_agent: + headers.append(('User-Agent', self.user_agent)) + for ehname, ehval in extra_headers: + headers.append((ehname, ehval)) + logging.debug('---- request headers ----') + for hname, hval in headers: + logging.debug('%s: %s', hname, hval) + h.putheader(hname, hval) + logging.debug('---- request body (query) ----') + logging.debug(query) + h.endheaders(query.encode()) res = h.getresponse() response = res.read().decode('ascii', 'ignore') logging.debug('---- response ----') logging.debug(res.__dict__) + logging.debug('Headers: %s', res.getheaders()) logging.debug(response) res.close() - - return response + return res, response def next_cookie(self): self.cookie += 1 diff --git a/ofxclient/config.py b/ofxclient/config.py index 0dfeaac..7a6757c 100644 --- a/ofxclient/config.py +++ b/ofxclient/config.py @@ -100,6 +100,8 @@ class SecurableConfigParser(ConfigParser): for k, v in ConfigParser.items(self, section): if self.is_secure_option(section, k): v = self.get(section, k) + if v == '!!False!!': + v = False items.append((k, v)) return items @@ -116,6 +118,8 @@ class SecurableConfigParser(ConfigParser): def set(self, section, option, value): """Set an option value. Knows how to set options properly marked as secure.""" + if not value: + value = '!!False!!' if self.is_secure_option(section, option): self.set_secure(section, option, value) else: @@ -139,10 +143,14 @@ class SecurableConfigParser(ConfigParser): if self.is_secure_option(section, option) and self.keyring_available: s_option = "%s%s" % (section, option) if self._unsaved.get(s_option, [''])[0] == 'set': - return self._unsaved[s_option][1] + res = self._unsaved[s_option][1] else: - return keyring.get_password(self.keyring_name, s_option) - return ConfigParser.get(self, section, option, *args) + res = keyring.get_password(self.keyring_name, s_option) + else: + res = ConfigParser.get(self, section, option, *args) + if res == '!!False!!': + return False + return res def remove_option(self, section, option): """Removes the option from ConfigParser as well as diff --git a/ofxclient/institution.py b/ofxclient/institution.py index 7e38096..3a06588 100644 --- a/ofxclient/institution.py +++ b/ofxclient/institution.py @@ -8,7 +8,7 @@ try: except ImportError: # python 2 from StringIO import StringIO - IS_PYTHON_2 = Trues + IS_PYTHON_2 = True from bs4 import BeautifulSoup from ofxparse import OfxParser @@ -145,7 +145,7 @@ class Institution(object): if IS_PYTHON_2: parsed = OfxParser.parse(resp_handle) else: - parsed = OfxParser.parse(BytesIO((((resp_handle).read()).encode()))) + parsed = OfxParser.parse(BytesIO(resp_handle.read().encode())) return [Account.from_ofxparse(a, institution=self) for a in parsed.accounts] @@ -165,22 +165,14 @@ class Institution(object): 'password': 'Customer password', 'description': 'descr', 'client_args': { - 'id': 'random client id - see Client() for default', - 'app_id': 'app name - see Client() for default', - 'app_version': 'app version - see Client() for default', - 'ofx_version': 'ofx version - see Client() for default', + 'id': 'random client id - see Client() for default', + 'app_id': 'app name - see Client() for default', + '...': 'see Client() for other options' } } :rtype: nested dictionary """ - client = self.client() - client_args = { - 'id': client.id, - 'app_id': client.app_id, - 'app_version': client.app_version, - 'ofx_version': client.ofx_version, - } return { 'id': self.id, 'org': self.org, @@ -189,7 +181,7 @@ class Institution(object): 'username': self.username, 'password': self.password, 'description': self.description, - 'client_args': client_args, + 'client_args': self.client().init_args, 'local_id': self.local_id() } diff --git a/ofxclient/version.py b/ofxclient/version.py index 18e8949..492f5aa 100644 --- a/ofxclient/version.py +++ b/ofxclient/version.py @@ -1,2 +1,2 @@ from __future__ import unicode_literals -__version__ = '2.0.2' +__version__ = '2.0.3' diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..8bb2683 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,3 @@ +-r requirements.txt +mock +nose diff --git a/requirements.txt b/requirements.txt index 5aea447..3ac7954 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -argparse ==1.4.1; python_version < '2.7' -beautifulsoup4 ==4.4.1 -keyring ==8.4.1 -ofxhome ==0.3.3 -ofxparse ==0.14 -lxml ==3.5.0 -keyrings.alt ==1.1.1 -pycrypto ==2.6.1 +argparse==1.4.1; python_version < '2.7' +beautifulsoup4==4.4.1 +keyring==8.4.1 +ofxhome==0.3.3 +ofxparse==0.14 +lxml>=3.5.0 +keyrings.alt==1.1.1 +pycrypto==2.6.1 diff --git a/setup.py b/setup.py index 696ab5f..3b77751 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ from setuptools import setup, find_packages -import os import re VERSIONFILE = "ofxclient/version.py" diff --git a/tests/ofxconfig.py b/tests/ofxconfig.py index fdbb251..8800146 100644 --- a/tests/ofxconfig.py +++ b/tests/ofxconfig.py @@ -1,7 +1,13 @@ +import keyring +from keyrings.alt.file import PlaintextKeyring import os import os.path import tempfile import unittest +try: + from test.support import EnvironmentVarGuard +except ImportError: + from test.test_support import EnvironmentVarGuard import ofxclient.config from ofxclient.config import OfxConfig @@ -11,8 +17,15 @@ from ofxclient import Institution, CreditCardAccount class OfxConfigTests(unittest.TestCase): def setUp(self): + keyring.set_keyring(PlaintextKeyring()) + + self.env = EnvironmentVarGuard() self.temp_file = tempfile.NamedTemporaryFile() + test_path = os.path.dirname(os.path.realpath(__file__)) + self.env['XDG_DATA_HOME'] = test_path + self.env['XDG_CONFIG_HOME'] = test_path + def tearDown(self): self.temp_file.close() @@ -22,7 +35,7 @@ class OfxConfigTests(unittest.TestCase): self.assertFalse(os.path.exists(file_name)) - c = OfxConfig(file_name=file_name) + c = OfxConfig(file_name=file_name) # noqa self.assertTrue(os.path.exists(file_name)) os.remove(file_name) @@ -72,6 +85,9 @@ class OfxConfigTests(unittest.TestCase): if not ofxclient.config.KEYRING_AVAILABLE: return + # always skip these for now + return + c = OfxConfig(file_name=self.temp_file.name) i = Institution( @@ -95,6 +111,9 @@ class OfxConfigTests(unittest.TestCase): if not ofxclient.config.KEYRING_AVAILABLE: return + # always skip these for now + return + c = OfxConfig(file_name=self.temp_file.name) i = Institution( id='1', @@ -123,6 +142,9 @@ class OfxConfigTests(unittest.TestCase): if not ofxclient.config.KEYRING_AVAILABLE: return + # always skip these for now + return + c = OfxConfig(file_name=self.temp_file.name) i = Institution( id='1', -- cgit v1.2.3