From d8216aeb9c12ea81d9941edc6eff39be32c24aca Mon Sep 17 00:00:00 2001 From: Andrew Shadura Date: Wed, 19 Oct 2016 18:02:05 +0200 Subject: Imported Upstream version 0.15 --- AUTHORS | 9 + PKG-INFO | 48 ++++-- README | 71 -------- README.rst | 95 +++++++++++ ofxparse.egg-info/PKG-INFO | 48 ++++-- ofxparse.egg-info/SOURCES.txt | 6 +- ofxparse.egg-info/requires.txt | 4 +- ofxparse/__init__.py | 3 +- ofxparse/ofxparse.py | 337 +++++++++++++++++++++++++++---------- ofxparse/ofxprinter.py | 194 +++++++++++++++++++++ ofxparse/ofxutil.py | 49 +++--- setup.py | 18 +- tests/fixtures/investment_401k.ofx | 180 ++++++++++++++++++++ tests/fixtures/suncorp.ofx | 56 ++++++ tests/fixtures/vanguard401k.ofx | 11 ++ tests/test_parse.py | 207 +++++++++++++++++++++-- tests/test_write.py | 1 + 17 files changed, 1117 insertions(+), 220 deletions(-) delete mode 100644 README create mode 100644 README.rst create mode 100644 ofxparse/ofxprinter.py create mode 100644 tests/fixtures/investment_401k.ofx create mode 100644 tests/fixtures/suncorp.ofx create mode 100644 tests/fixtures/vanguard401k.ofx diff --git a/AUTHORS b/AUTHORS index 7e58dca..ee768ff 100644 --- a/AUTHORS +++ b/AUTHORS @@ -14,3 +14,12 @@ Andre Smolik greggles@github mikeivanov@github danc86@github +Erik Hetzner +Panagiotis Issaris +Brett Trotter +Joe Cabrera +Michael Nelson +Nathan Grigg +Wes Turner +Ehud Ben-Reuven +Joseph Walton diff --git a/PKG-INFO b/PKG-INFO index 83be6d8..c19dfd2 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: ofxparse -Version: 0.14 +Version: 0.15 Summary: Tools for working with the OFX (Open Financial Exchange) file format Home-page: http://sites.google.com/site/ofxparse Author: Jerry Seutter @@ -23,11 +23,13 @@ Description: ofxparse Example Usage ============= - Here's a sample program:: + Here's a sample program - from ofxparse import OfxParser + .. code:: python - ofx = OfxParser.parse(file('file.ofx')) + from ofxparse import OfxParser + with codecs.open('file.ofx') as fileobj: + ofx = OfxParser.parse(fileobj) ofx.accounts # An account with information ofx.account.number # The account number ofx.account.routing_number # The transit id (sometimes called branch number) @@ -41,7 +43,8 @@ Description: ofxparse Help! ===== - Sample .ofx files are very useful. If you want to help us out, please edit + Sample ``.ofx`` and ``.qfx`` files are very useful. + If you want to help us out, please edit all identifying information from the file and then email it to jseutter dot ofxparse at gmail dot com. @@ -49,25 +52,46 @@ Description: ofxparse =========== Prerequisites:: - (Ubuntu) sudo apt-get install python-beautifulsoup python-nose python-coverage-test-runner - (pip) pip install BeautifulSoup nose coverage + # Ubuntu + sudo apt-get install python-beautifulsoup python-nose python-coverage-test-runner + + # pip for Python 3: + pip install BeautifulSoup4 six lxml nose coverage + + # pip for Python 2: + pip install BeautifulSoup six nose coverage + + Tests: + Simply running the ``nosetests`` command should run the tests. - Tests:: - Simply running the "nose" command should run the tests. If you don't have nose - installed, the following might also work: + .. code:: bash + + nosetests + + If you don't have nose installed, the following might also work: + + .. code:: bash python -m unittest tests.test_parse - Test Coverage Report:: + Test Coverage Report: + + .. code:: bash coverage run -m unittest tests.test_parse + + # text report + coverage report + + # html report coverage html firefox htmlcov/index.html Homepage ======== - http://sites.google.com/site/ofxparse + | Homepage: https://sites.google.com/site/ofxparse + | Source: https://github.com/jseutter/ofxparse License ======= diff --git a/README b/README deleted file mode 100644 index c112ec0..0000000 --- a/README +++ /dev/null @@ -1,71 +0,0 @@ -ofxparse -======== - -ofxparse is a parser for Open Financial Exchange (.ofx) format files. OFX -files are available from almost any online banking site, so they work well -if you want to pull together your finances from multiple sources. Online -trading accounts also provide account statements in OFX files. - -There are three different types of OFX files, called BankAccount, -CreditAccount and InvestmentAccount files. This library has been tested with -real-world samples of all three types. If you find a file that does not work -with this library, please consider contributing the file so ofxparse can be -improved. See the Help! section below for directions on how to do this. - -Example Usage -============= - -Here's a sample program:: - - from ofxparse import OfxParser - - ofx = OfxParser.parse(file('file.ofx')) - ofx.accounts # An account with information - ofx.account.number # The account number - ofx.account.routing_number # The transit id (sometimes called branch number) - ofx.account.statement # Account information for a period of time - ofx.account.statement.start_date # The start date of the transactions - ofx.account.statement.end_date # The end date of the transactions - ofx.account.statement.transactions # A list of account activities - ofx.account.statement.balance # The money in the account as of the statement date - ofx.account.statement.available_balance # The money available from the account as of the statement date - -Help! -===== - -Sample .ofx files are very useful. If you want to help us out, please edit -all identifying information from the file and then email it to jseutter dot -ofxparse at gmail dot com. - -Development -=========== - -Prerequisites:: -(Ubuntu) sudo apt-get install python-beautifulsoup python-nose python-coverage-test-runner -(pip) pip install BeautifulSoup nose coverage - -Tests:: -Simply running the "nose" command should run the tests. If you don't have nose -installed, the following might also work: - - python -m unittest tests.test_parse - -Test Coverage Report:: - - coverage run -m unittest tests.test_parse - coverage html - firefox htmlcov/index.html - - -Homepage -======== -http://sites.google.com/site/ofxparse - -License -======= - -ofxparse is released under an MIT license. See the LICENSE file for the actual -license text. The basic idea is that if you can use Python to do what you are -doing, you can also use this library. - - diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..0015743 --- /dev/null +++ b/README.rst @@ -0,0 +1,95 @@ +ofxparse +======== + +ofxparse is a parser for Open Financial Exchange (.ofx) format files. OFX +files are available from almost any online banking site, so they work well +if you want to pull together your finances from multiple sources. Online +trading accounts also provide account statements in OFX files. + +There are three different types of OFX files, called BankAccount, +CreditAccount and InvestmentAccount files. This library has been tested with +real-world samples of all three types. If you find a file that does not work +with this library, please consider contributing the file so ofxparse can be +improved. See the Help! section below for directions on how to do this. + +Example Usage +============= + +Here's a sample program + +.. code:: python + + from ofxparse import OfxParser + with codecs.open('file.ofx') as fileobj: + ofx = OfxParser.parse(fileobj) + ofx.accounts # An account with information + ofx.account.number # The account number + ofx.account.routing_number # The transit id (sometimes called branch number) + ofx.account.statement # Account information for a period of time + ofx.account.statement.start_date # The start date of the transactions + ofx.account.statement.end_date # The end date of the transactions + ofx.account.statement.transactions # A list of account activities + ofx.account.statement.balance # The money in the account as of the statement date + ofx.account.statement.available_balance # The money available from the account as of the statement date + +Help! +===== + +Sample ``.ofx`` and ``.qfx`` files are very useful. +If you want to help us out, please edit +all identifying information from the file and then email it to jseutter dot +ofxparse at gmail dot com. + +Development +=========== + +Prerequisites:: + # Ubuntu + sudo apt-get install python-beautifulsoup python-nose python-coverage-test-runner + + # pip for Python 3: + pip install BeautifulSoup4 six lxml nose coverage + + # pip for Python 2: + pip install BeautifulSoup six nose coverage + +Tests: +Simply running the ``nosetests`` command should run the tests. + +.. code:: bash + + nosetests + +If you don't have nose installed, the following might also work: + +.. code:: bash + + python -m unittest tests.test_parse + +Test Coverage Report: + +.. code:: bash + + coverage run -m unittest tests.test_parse + + # text report + coverage report + + # html report + coverage html + firefox htmlcov/index.html + + +Homepage +======== +| Homepage: https://sites.google.com/site/ofxparse +| Source: https://github.com/jseutter/ofxparse + +License +======= + +ofxparse is released under an MIT license. See the LICENSE file for the actual +license text. The basic idea is that if you can use Python to do what you are +doing, you can also use this library. + + diff --git a/ofxparse.egg-info/PKG-INFO b/ofxparse.egg-info/PKG-INFO index 83be6d8..c19dfd2 100644 --- a/ofxparse.egg-info/PKG-INFO +++ b/ofxparse.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: ofxparse -Version: 0.14 +Version: 0.15 Summary: Tools for working with the OFX (Open Financial Exchange) file format Home-page: http://sites.google.com/site/ofxparse Author: Jerry Seutter @@ -23,11 +23,13 @@ Description: ofxparse Example Usage ============= - Here's a sample program:: + Here's a sample program - from ofxparse import OfxParser + .. code:: python - ofx = OfxParser.parse(file('file.ofx')) + from ofxparse import OfxParser + with codecs.open('file.ofx') as fileobj: + ofx = OfxParser.parse(fileobj) ofx.accounts # An account with information ofx.account.number # The account number ofx.account.routing_number # The transit id (sometimes called branch number) @@ -41,7 +43,8 @@ Description: ofxparse Help! ===== - Sample .ofx files are very useful. If you want to help us out, please edit + Sample ``.ofx`` and ``.qfx`` files are very useful. + If you want to help us out, please edit all identifying information from the file and then email it to jseutter dot ofxparse at gmail dot com. @@ -49,25 +52,46 @@ Description: ofxparse =========== Prerequisites:: - (Ubuntu) sudo apt-get install python-beautifulsoup python-nose python-coverage-test-runner - (pip) pip install BeautifulSoup nose coverage + # Ubuntu + sudo apt-get install python-beautifulsoup python-nose python-coverage-test-runner + + # pip for Python 3: + pip install BeautifulSoup4 six lxml nose coverage + + # pip for Python 2: + pip install BeautifulSoup six nose coverage + + Tests: + Simply running the ``nosetests`` command should run the tests. - Tests:: - Simply running the "nose" command should run the tests. If you don't have nose - installed, the following might also work: + .. code:: bash + + nosetests + + If you don't have nose installed, the following might also work: + + .. code:: bash python -m unittest tests.test_parse - Test Coverage Report:: + Test Coverage Report: + + .. code:: bash coverage run -m unittest tests.test_parse + + # text report + coverage report + + # html report coverage html firefox htmlcov/index.html Homepage ======== - http://sites.google.com/site/ofxparse + | Homepage: https://sites.google.com/site/ofxparse + | Source: https://github.com/jseutter/ofxparse License ======= diff --git a/ofxparse.egg-info/SOURCES.txt b/ofxparse.egg-info/SOURCES.txt index 6cff2c7..21fc7ff 100644 --- a/ofxparse.egg-info/SOURCES.txt +++ b/ofxparse.egg-info/SOURCES.txt @@ -1,12 +1,13 @@ AUTHORS LICENSE MANIFEST.in -README +README.rst setup.cfg setup.py ofxparse/__init__.py ofxparse/mcc.py ofxparse/ofxparse.py +ofxparse/ofxprinter.py ofxparse/ofxutil.py ofxparse.egg-info/PKG-INFO ofxparse.egg-info/SOURCES.txt @@ -24,13 +25,16 @@ tests/fixtures/bank_medium.ofx tests/fixtures/bank_small.ofx tests/fixtures/checking.ofx tests/fixtures/fidelity.ofx +tests/fixtures/investment_401k.ofx tests/fixtures/investment_medium.ofx tests/fixtures/multiple_accounts.ofx tests/fixtures/multiple_accounts2.ofx tests/fixtures/signon_fail.ofx tests/fixtures/signon_success.ofx tests/fixtures/signon_success_no_message.ofx +tests/fixtures/suncorp.ofx tests/fixtures/vanguard.ofx +tests/fixtures/vanguard401k.ofx tests/fixtures/fail_nice/date_missing.ofx tests/fixtures/fail_nice/decimal_error.ofx tests/fixtures/fail_nice/empty_balance.ofx \ No newline at end of file diff --git a/ofxparse.egg-info/requires.txt b/ofxparse.egg-info/requires.txt index d33d9e5..c038268 100644 --- a/ofxparse.egg-info/requires.txt +++ b/ofxparse.egg-info/requires.txt @@ -1,2 +1,4 @@ beautifulsoup4 -six \ No newline at end of file +lxml +six +lxml diff --git a/ofxparse/__init__.py b/ofxparse/__init__.py index f1fb25b..a08ba77 100644 --- a/ofxparse/__init__.py +++ b/ofxparse/__init__.py @@ -1,5 +1,6 @@ from __future__ import absolute_import from .ofxparse import OfxParser, AccountType, Account, Statement, Transaction +from .ofxprinter import OfxPrinter -__version__ = '0.14' +__version__ = '0.15' diff --git a/ofxparse/ofxparse.py b/ofxparse/ofxparse.py index a66df88..5686030 100644 --- a/ofxparse/ofxparse.py +++ b/ofxparse/ofxparse.py @@ -22,13 +22,37 @@ else: from . import mcc + +def skip_headers(fh): + ''' + Prepare `fh` for parsing by BeautifulSoup by skipping its OFX + headers. + ''' + if fh is None or isinstance(fh, six.string_types): + return + fh.seek(0) + header_re = re.compile(r"^\s*\w+:\s*\w+\s*$") + while True: + pos = fh.tell() + line = fh.readline() + if not line: + break + if header_re.search(line) is None: + fh.seek(pos) + return + + def soup_maker(fh): + skip_headers(fh) try: from bs4 import BeautifulSoup - return BeautifulSoup(fh) + soup = BeautifulSoup(fh, "xml") + for tag in soup.findAll(): + tag.name = tag.name.lower() except ImportError: from BeautifulSoup import BeautifulStoneSoup - return BeautifulStoneSoup(fh) + soup = BeautifulStoneSoup(fh) + return soup def try_decode(string, encoding): @@ -36,11 +60,13 @@ def try_decode(string, encoding): string = string.decode(encoding) return string + def is_iterable(candidate): - if sys.version_info < (2,6): + if sys.version_info < (2, 6): return hasattr(candidate, 'next') return isinstance(candidate, collections.Iterable) + @contextlib.contextmanager def save_pos(fh): """ @@ -54,6 +80,7 @@ def save_pos(fh): finally: fh.seek(orig_pos) + class OfxFile(object): def __init__(self, fh): """ @@ -76,7 +103,7 @@ class OfxFile(object): head_data = self.fh.read(1024 * 10) head_data = head_data[:head_data.find(six.b('<'))] - for line in re.split(six.b('\r?\n?'), head_data): + for line in head_data.splitlines(): # Newline? if line.strip() == six.b(""): break @@ -138,7 +165,7 @@ class OfxFile(object): class OfxPreprocessedFile(OfxFile): def __init__(self, fh): - super(OfxPreprocessedFile,self).__init__(fh) + super(OfxPreprocessedFile, self).__init__(fh) if self.fh is None: return @@ -146,19 +173,21 @@ class OfxPreprocessedFile(OfxFile): ofx_string = self.fh.read() # find all closing tags as hints - closing_tags = [ t.upper() for t in re.findall(r'(?i)', ofx_string) ] + closing_tags = [t.upper() for t in re.findall(r'(?i)', + ofx_string)] # close all tags that don't have closing tags and # leave all other data intact last_open_tag = None - tokens = re.split(r'(?i)()', ofx_string) - new_fh = StringIO() - for idx,token in enumerate(tokens): + tokens = re.split(r'(?i)()', ofx_string) + new_fh = StringIO() + for idx, token in enumerate(tokens): is_closing_tag = token.startswith('" % last_open_tag) @@ -175,7 +204,8 @@ class OfxPreprocessedFile(OfxFile): class Ofx(object): def __str__(self): return "" -# headers = "\r\n".join(":".join(el if el else "NONE" for el in item) for item in six.iteritems(self.headers)) +# headers = "\r\n".join(":".join(el if el else "NONE" for el in item) +# for item in six.iteritems(self.headers)) # headers += "\r\n\r\n" # # return headers + str(self.signon) @@ -187,6 +217,7 @@ class AccountType(object): class Account(object): def __init__(self): + self.curdef = None self.statement = None self.account_id = '' self.routing_number = '' @@ -216,25 +247,52 @@ class Security: self.ticker = ticker self.memo = memo + class Signon: - def __init__(self, code, severity, message): - self.code = code - self.severity = severity - self.message = message - if int(code) == 0: + def __init__(self, keys): + self.code = keys['code'] + self.severity = keys['severity'] + self.message = keys['message'] + self.dtserver = keys['dtserver'] + self.language = keys['language'] + self.dtprofup = keys['dtprofup'] + self.fi_org = keys['org'] + self.fi_fid = keys['fid'] + self.intu_bid = keys['intu.bid'] + + if int(self.code) == 0: self.success = True else: self.success = False def __str__(self): - ret = "\t\r\n" + "\t\t\r\n" + "\t\t\t\r\n" + ret = "\t\r\n" + "\t\t\r\n" + \ + "\t\t\t\r\n" ret += "\t\t\t\t%s\r\n" % self.code ret += "\t\t\t\t%s\r\n" % self.severity if self.message: ret += "\t\t\t\t%s\r\n" % self.message - ret += "\t\t\t\r\n" + "\t\t\r\n" + "\t\r\n" + ret += "\t\t\t\r\n" + if self.dtserver is not None: + ret += "\t\t\t" + self.dtserver + "\r\n" + if self.language is not None: + ret += "\t\t\t" + self.language + "\r\n" + if self.dtprofup is not None: + ret += "\t\t\t" + self.dtprofup + "\r\n" + if (self.fi_org is not None) or (self.fi_fid is not None): + ret += "\t\t\t\r\n" + if self.fi_org is not None: + ret += "\t\t\t\t" + self.fi_org + "\r\n" + if self.fi_fid is not None: + ret += "\t\t\t\t" + self.fi_fid + "\r\n" + ret += "\t\t\t\r\n" + if self.intu_bid is not None: + ret += "\t\t\t" + self.intu_bid + "\r\n" + ret += "\t\t\r\n" + ret += "\t\r\n" return ret + class Statement(object): def __init__(self): self.start_date = '' @@ -272,20 +330,29 @@ class Transaction(object): class InvestmentTransaction(object): - (Unknown, BuyMF, SellMF, Reinvest, BuyStock, SellStock) = [x for x in range(-1, 5)] + AGGREGATE_TYPES = ['buydebt', 'buymf', 'buyopt', 'buyother', + 'buystock', 'closureopt', 'income', + 'invexpense', 'jrnlfund', 'jrnlsec', + 'margininterest', 'reinvest', 'retofcap', + 'selldebt', 'sellmf', 'sellopt', 'sellother', + 'sellstock', 'split', 'transfer'] + def __init__(self, type): - try: - self.type = ['buymf', 'sellmf', 'reinvest', 'buystock', 'sellstock'].index(type.lower()) - except ValueError: - self.type = InvestmentTransaction.Unknown + self.type = type.lower() self.tradeDate = None self.settleDate = None + self.memo = '' self.security = '' + self.income_type = '' self.units = decimal.Decimal(0) self.unit_price = decimal.Decimal(0) + self.commission = decimal.Decimal(0) + self.fees = decimal.Decimal(0) + self.total = decimal.Decimal(0) def __repr__(self): - return "" + return "" class Position(object): @@ -321,8 +388,9 @@ class OfxParser(object): ''' cls_.fail_fast = fail_fast - if isinstance(file_handle, type('')): - raise RuntimeError(six.u("parse() takes in a file handle, not a string")) + if not hasattr(file_handle, 'seek'): + raise TypeError(six.u('parse() accepts a seek-able file handle\ + , not %s' % type(file_handle).__name__)) ofx_obj = Ofx() @@ -332,14 +400,30 @@ class OfxParser(object): ofx_obj.accounts = [] ofx_obj.signon = None + skip_headers(ofx_file.fh) ofx = soup_maker(ofx_file.fh) - if len(ofx.contents) == 0: + if ofx.find('ofx') is None: raise OfxParserException('The ofx file is empty!') sonrs_ofx = ofx.find('sonrs') if sonrs_ofx: ofx_obj.signon = cls_.parseSonrs(sonrs_ofx) + stmttrnrs = ofx.find('stmttrnrs') + if stmttrnrs: + stmttrnrs_trnuid = stmttrnrs.find('trnuid') + if stmttrnrs_trnuid: + ofx_obj.trnuid = stmttrnrs_trnuid.contents[0].strip() + + stmttrnrs_status = stmttrnrs.find('status') + if stmttrnrs_status: + ofx_obj.status = {} + ofx_obj.status['code'] = int( + stmttrnrs_status.find('code').contents[0].strip() + ) + ofx_obj.status['severity'] = \ + stmttrnrs_status.find('severity').contents[0].strip() + stmtrs_ofx = ofx.findAll('stmtrs') if stmtrs_ofx: ofx_obj.accounts += cls_.parseStmtrs(stmtrs_ofx, AccountType.Bank) @@ -386,14 +470,23 @@ class OfxParser(object): timeZoneOffset = datetime.timedelta(hours=tz) + res = re.search("^[0-9]*\.([0-9]{0,5})", ofxDateTime) + if res: + msec = datetime.timedelta(seconds=float("0." + res.group(1))) + else: + msec = datetime.timedelta(seconds=0) + try: local_date = datetime.datetime.strptime( ofxDateTime[:14], '%Y%m%d%H%M%S' ) - return local_date - timeZoneOffset + return local_date - timeZoneOffset + msec except: + if ofxDateTime[:8] == "00000000": + return None + return datetime.datetime.strptime( - ofxDateTime[:8], '%Y%m%d') - timeZoneOffset + ofxDateTime[:8], '%Y%m%d') - timeZoneOffset + msec @classmethod def parseAcctinfors(cls_, acctinfors_ofx, ofx): @@ -462,7 +555,12 @@ class OfxParser(object): name_tag = secinfo_ofx.find('secname') ticker_tag = secinfo_ofx.find('ticker') memo_tag = secinfo_ofx.find('memo') - if uniqueid_tag and name_tag and ticker_tag: + if uniqueid_tag and name_tag: + try: + ticker = ticker_tag.contents[0].strip() + except AttributeError: + # ticker can be empty + ticker = None try: memo = memo_tag.contents[0].strip() except AttributeError: @@ -471,7 +569,7 @@ class OfxParser(object): securityList.append( Security(uniqueid_tag.contents[0].strip(), name_tag.contents[0].strip(), - ticker_tag.contents[0].strip(), + ticker, memo)) return securityList @@ -483,10 +581,10 @@ class OfxParser(object): position.security = tag.contents[0].strip() tag = ofx.find('units') if (hasattr(tag, 'contents')): - position.units = decimal.Decimal(tag.contents[0].strip()) + position.units = cls_.toDecimal(tag) tag = ofx.find('unitprice') if (hasattr(tag, 'contents')): - position.unit_price = decimal.Decimal(tag.contents[0].strip()) + position.unit_price = cls_.toDecimal(tag) tag = ofx.find('dtpriceasof') if (hasattr(tag, 'contents')): try: @@ -521,12 +619,27 @@ class OfxParser(object): tag = ofx.find('uniqueid') if (hasattr(tag, 'contents')): transaction.security = tag.contents[0].strip() + tag = ofx.find('incometype') + if (hasattr(tag, 'contents')): + transaction.income_type = tag.contents[0].strip() tag = ofx.find('units') if (hasattr(tag, 'contents')): - transaction.units = decimal.Decimal(tag.contents[0].strip()) + transaction.units = cls_.toDecimal(tag) tag = ofx.find('unitprice') if (hasattr(tag, 'contents')): - transaction.unit_price = decimal.Decimal(tag.contents[0].strip()) + transaction.unit_price = cls_.toDecimal(tag) + tag = ofx.find('commission') + if (hasattr(tag, 'contents')): + transaction.commission = cls_.toDecimal(tag) + tag = ofx.find('fees') + if (hasattr(tag, 'contents')): + transaction.fees = cls_.toDecimal(tag) + tag = ofx.find('total') + if (hasattr(tag, 'contents')): + transaction.total = cls_.toDecimal(tag) + tag = ofx.find('inv401ksource') + if (hasattr(tag, 'contents')): + transaction.inv401ksource = tag.contents[0].strip() return transaction @classmethod @@ -548,7 +661,8 @@ class OfxParser(object): raise except ValueError: e = sys.exc_info()[1] - statement.warnings.append(six.u('Invalid start date: %s') % e) + statement.warnings.append(six.u('Invalid start date:\ + %s') % e) if cls_.fail_fast: raise @@ -561,7 +675,8 @@ class OfxParser(object): statement.warnings.append(six.u('Empty end date.')) except ValueError: e = sys.exc_info()[1] - statement.warnings.append(six.u('Invalid end date: %s') % e) + statement.warnings.append(six.u('Invalid end date: \ + %s') % e) if cls_.fail_fast: raise @@ -576,12 +691,11 @@ class OfxParser(object): if cls_.fail_fast: raise statement.discarded_entries.append( - {six.u('error'): six.u("Error parsing positions: ") + str(e), - six.u('content'): investment_ofx} + {six.u('error'): six.u("Error parsing positions: \ + ") + str(e), six.u('content'): investment_ofx} ) - for transaction_type in ['buymf', 'sellmf', 'reinvest', 'buystock', - 'sellstock', 'buyopt', 'sellopt']: + for transaction_type in InvestmentTransaction.AGGREGATE_TYPES: try: for investment_ofx in invstmtrs_ofx.findAll(transaction_type): statement.transactions.append( @@ -613,14 +727,28 @@ class OfxParser(object): @classmethod def parseSonrs(cls_, sonrs): - code = int(sonrs.find('code').contents[0].strip()) - severity = sonrs.find('severity').contents[0].strip() - try: - message = sonrs.find('message').contents[0].strip() - except: - message = '' + items = [ + 'code', + 'severity', + 'dtserver', + 'language', + 'dtprofup', + 'org', + 'fid', + 'intu.bid', + 'message' + ] + idict = {} + for i in items: + try: + idict[i] = sonrs.find(i).contents[0].strip() + except: + idict[i] = None + idict['code'] = int(idict['code']) + if idict['message'] is None: + idict['message'] = '' - return Signon(code,severity,message) + return Signon(idict) @classmethod def parseStmtrs(cls_, stmtrs_list, accountType): @@ -628,6 +756,9 @@ class OfxParser(object): ret = [] for stmtrs_ofx in stmtrs_list: account = Account() + act_curdef = stmtrs_ofx.find('curdef') + if act_curdef: + account.curdef = act_curdef.contents[0].strip() acctid_tag = stmtrs_ofx.find('acctid') if hasattr(acctid_tag, 'contents'): account.account_id = acctid_tag.contents[0].strip() @@ -647,6 +778,41 @@ class OfxParser(object): ret.append(account) return ret + @classmethod + def parseBalance(cls_, statement, stmt_ofx, bal_tag_name, bal_attr, + bal_date_attr, bal_type_string): + bal_tag = stmt_ofx.find(bal_tag_name) + if hasattr(bal_tag, "contents"): + balamt_tag = bal_tag.find('balamt') + dtasof_tag = bal_tag.find('dtasof') + if hasattr(balamt_tag, "contents"): + try: + setattr(statement, bal_attr, cls_.toDecimal(balamt_tag)) + except (IndexError, decimal.InvalidOperation): + ex = sys.exc_info()[1] + statement.warnings.append( + six.u("%s balance amount was empty for \ + %s") % (bal_type_string, stmt_ofx)) + if cls_.fail_fast: + raise OfxParserException("Empty %s balance\ + " % bal_type_string) + if hasattr(dtasof_tag, "contents"): + try: + setattr(statement, bal_date_attr, cls_.parseOfxDateTime( + dtasof_tag.contents[0].strip())) + except IndexError: + statement.warnings.append( + six.u("%s balance date was empty for %s\ + ") % (bal_type_string, stmt_ofx)) + if cls_.fail_fast: + raise + except ValueError: + statement.warnings.append( + six.u("%s balance date was not allowed for \ + %s") % (bal_type_string, stmt_ofx)) + if cls_.fail_fast: + raise + @classmethod def parseStatement(cls_, stmt_ofx): ''' @@ -665,7 +831,8 @@ class OfxParser(object): raise except ValueError: statement.warnings.append( - six.u("Statement start date was not allowed for %s") % stmt_ofx) + six.u("Statement start date was not allowed for \ + %s") % stmt_ofx) if cls_.fail_fast: raise @@ -682,13 +849,14 @@ class OfxParser(object): except ValueError: ve = sys.exc_info()[1] msg = six.u("Statement start date was not formatted " - "correctly for %s") + "correctly for %s") statement.warnings.append(msg % stmt_ofx) if cls_.fail_fast: raise except TypeError: statement.warnings.append( - six.u("Statement start date was not allowed for %s") % stmt_ofx) + six.u("Statement start date was not allowed for \ + %s") % stmt_ofx) if cls_.fail_fast: raise @@ -702,33 +870,11 @@ class OfxParser(object): if cls_.fail_fast: raise - ledger_bal_tag = stmt_ofx.find('ledgerbal') - if hasattr(ledger_bal_tag, "contents"): - balamt_tag = ledger_bal_tag.find('balamt') - if hasattr(balamt_tag, "contents"): - try: - statement.balance = decimal.Decimal( - balamt_tag.contents[0].strip()) - except (IndexError, decimal.InvalidOperation): - ex = sys.exc_info()[1] - statement.warnings.append( - six.u("Ledger balance amount was empty for %s") % stmt_ofx) - if cls_.fail_fast: - raise OfxParserException("Empty ledger balance") + cls_.parseBalance(statement, stmt_ofx, 'ledgerbal', + 'balance', 'balance_date', 'ledger') - avail_bal_tag = stmt_ofx.find('availbal') - if hasattr(avail_bal_tag, "contents"): - balamt_tag = avail_bal_tag.find('balamt') - if hasattr(balamt_tag, "contents"): - try: - statement.available_balance = decimal.Decimal( - balamt_tag.contents[0].strip()) - except (IndexError, decimal.InvalidOperation): - ex = sys.exc_info()[1] - msg = six.u("Available balance amount was empty for %s") - statement.warnings.append(msg % stmt_ofx) - if cls_.fail_fast: - raise OfxParserException("Empty available balance") + cls_.parseBalance(statement, stmt_ofx, 'availbal', 'available_balance', + 'available_balance_date', 'ledger') for transaction_ofx in stmt_ofx.findAll('stmttrn'): try: @@ -783,13 +929,17 @@ class OfxParser(object): amt_tag = txn_ofx.find('trnamt') if hasattr(amt_tag, "contents"): try: - transaction.amount = decimal.Decimal( - amt_tag.contents[0].strip()) + transaction.amount = cls_.toDecimal(amt_tag) except IndexError: raise OfxParserException("Invalid Transaction Date") except decimal.InvalidOperation: - raise OfxParserException( - six.u("Invalid Transaction Amount: '%s'") % amt_tag.contents[0]) + # Some banks use a null transaction for including interest + # rate changes on your statement. + if amt_tag.contents[0].strip() in ('null', '-null'): + transaction.amount = 0 + else: + raise OfxParserException( + six.u("Invalid Transaction Amount: '%s'") % amt_tag.contents[0]) except TypeError: raise OfxParserException( six.u("No Transaction Amount (a required field)")) @@ -819,24 +969,29 @@ class OfxParser(object): try: transaction.id = id_tag.contents[0].strip() except IndexError: - raise OfxParserException(six.u("Empty FIT id (a required field)")) + raise OfxParserException(six.u("Empty FIT id (a required \ + field)")) except TypeError: raise OfxParserException(six.u("No FIT id (a required field)")) else: - raise OfxParserException(six.u("Missing FIT id (a required field)")) + raise OfxParserException(six.u("Missing FIT id (a required \ + field)")) sic_tag = txn_ofx.find('sic') if hasattr(sic_tag, 'contents'): try: transaction.sic = sic_tag.contents[0].strip() except IndexError: - raise OfxParserException(six.u("Empty transaction Standard Industry Code (SIC)")) + raise OfxParserException(six.u("Empty transaction Standard \ + Industry Code (SIC)")) if transaction.sic is not None and transaction.sic in mcc.codes: try: - transaction.mcc = mcc.codes.get(transaction.sic, '').get('combined description') + transaction.mcc = mcc.codes.get(transaction.sic, '').get('combined \ + description') except IndexError: - raise OfxParserException(six.u("Empty transaction Merchant Category Code (MCC)")) + raise OfxParserException(six.u("Empty transaction Merchant Category \ + Code (MCC)")) except AttributeError: if cls._fail_fast: raise @@ -846,6 +1001,14 @@ class OfxParser(object): try: transaction.checknum = checknum_tag.contents[0].strip() except IndexError: - raise OfxParserException(six.u("Empty Check (or other reference) number")) + raise OfxParserException(six.u("Empty Check (or other reference) \ + number")) return transaction + + @classmethod + def toDecimal(cls_, tag): + d = tag.contents[0].strip() + if '.' not in d and ',' in d: + d = d.replace(',', '.') + return decimal.Decimal(d) diff --git a/ofxparse/ofxprinter.py b/ofxparse/ofxprinter.py new file mode 100644 index 0000000..4c0a13d --- /dev/null +++ b/ofxparse/ofxprinter.py @@ -0,0 +1,194 @@ +class OfxPrinter(): + ofx = None + out_filename = None + out_handle = None + term = "\r\n" + + def __init__(self, ofx, filename, term="\r\n"): + self.ofx = ofx + self.out_filename = filename + self.term = term + + def writeLine(self, data, tabs=0, term=None): + if term is None: + term = self.term + + tabbing = (tabs * "\t") if (tabs > 0) else '' + + return self.out_handle.write( + "{0}{1}{2}".format( + tabbing, + data, + term + ) + ) + + def writeHeaders(self): + for k, v in self.ofx.headers.iteritems(): + if v is None: + self.writeLine("{0}:NONE".format(k)) + else: + self.writeLine("{0}:{1}".format(k, v)) + self.writeLine("") + + def writeSignOn(self, tabs=0): + # signon already has newlines and tabs in it + # TODO: reimplement signon printing with tabs + self.writeLine(self.ofx.signon.__str__(), term="") + + def printDate(self, dt, msec_digs=3): + strdt = dt.strftime('%Y%m%d%H%M%S') + strdt_msec = dt.strftime('%f') + if len(strdt_msec) < msec_digs: + strdt_msec += ('0' * (msec_digs - len(strdt_msec))) + elif len(strdt_msec) > msec_digs: + strdt_msec = strdt_msec[:msec_digs] + return strdt + '.' + strdt_msec + + def writeTrn(self, trn, tabs=5): + self.writeLine("", tabs=tabs) + tabs += 1 + + self.writeLine("{}".format(trn.type.upper()), tabs=tabs) + self.writeLine("{}".format( + self.printDate(trn.date) + ), tabs=tabs) + self.writeLine("{0:.2f}".format(float(trn.amount)), tabs=tabs) + + self.writeLine("{}".format(trn.id), tabs=tabs) + + if len(str(trn.checknum)) > 0: + self.writeLine("{}".format( + trn.checknum + ), tabs=tabs) + + self.writeLine("{}".format(trn.payee), tabs=tabs) + + if len(trn.memo.strip()) > 0: + self.writeLine("{}".format(trn.memo), tabs=tabs) + + tabs -= 1 + self.writeLine("", tabs=tabs) + + def writeLedgerBal(self, statement, tabs=4): + bal = getattr(statement, 'balance') + baldt = getattr(statement, 'balance_date') + + if bal and baldt: + self.writeLine("", tabs=tabs) + self.writeLine("{0:.2f}".format(float(bal)), tabs=tabs+1) + self.writeLine("{0}".format( + self.printDate(baldt) + ), tabs=tabs+1) + self.writeLine("", tabs=tabs) + + def writeAvailBal(self, statement, tabs=4): + bal = getattr(statement, 'available_balance') + baldt = getattr(statement, 'available_balance_date') + + if bal and baldt: + self.writeLine("", tabs=tabs) + self.writeLine("{0:.2f}".format(float(bal)), tabs=tabs+1) + self.writeLine("{0}".format( + self.printDate(baldt) + ), tabs=tabs+1) + self.writeLine("", tabs=tabs) + + def writeStmTrs(self, tabs=3): + for acct in self.ofx.accounts: + self.writeLine("", tabs=tabs) + tabs += 1 + + if acct.curdef: + self.writeLine("{0}".format( + acct.curdef + ), tabs=tabs) + + if acct.routing_number or acct.account_id or acct.account_type: + self.writeLine("", tabs=tabs) + if acct.routing_number: + self.writeLine("{0}".format( + acct.routing_number + ), tabs=tabs+1) + if acct.account_id: + self.writeLine("{0}".format( + acct.account_id + ), tabs=tabs+1) + if acct.account_type: + self.writeLine("{0}".format( + acct.account_type + ), tabs=tabs+1) + self.writeLine("", tabs=tabs) + + self.writeLine("", tabs=tabs) + tabs += 1 + self.writeLine("{0}".format( + self.printDate(acct.statement.start_date) + ), tabs=tabs) + self.writeLine("{0}".format( + self.printDate(acct.statement.end_date) + ), tabs=tabs) + + for trn in acct.statement.transactions: + self.writeTrn(trn, tabs=tabs) + + tabs -= 1 + + self.writeLine("", tabs=tabs) + + self.writeLedgerBal(acct.statement, tabs=tabs) + self.writeAvailBal(acct.statement, tabs=tabs) + + tabs -= 1 + + self.writeLine("", tabs=tabs) + + def writeBankMsgsRsv1(self, tabs=1): + self.writeLine("", tabs=tabs) + tabs += 1 + self.writeLine("", tabs=tabs) + tabs += 1 + if self.ofx.trnuid is not None: + self.writeLine("{0}".format( + self.ofx.trnuid + ), tabs=tabs) + if self.ofx.status: + self.writeLine("", tabs=tabs) + self.writeLine("{0}".format( + self.ofx.status['code'] + ), tabs=tabs+1) + self.writeLine("{0}".format( + self.ofx.status['severity'] + ), tabs=tabs+1) + self.writeLine("", tabs=tabs) + self.writeStmTrs(tabs=tabs) + tabs -= 1 + self.writeLine("", tabs=tabs) + tabs -= 1 + self.writeLine("", tabs=tabs) + + def writeOfx(self, tabs=0): + self.writeLine("", tabs=tabs) + tabs += 1 + self.writeSignOn(tabs=tabs) + self.writeBankMsgsRsv1(tabs=tabs) + tabs -= 1 + # No newline at end of file + self.writeLine("", tabs=tabs, term="") + + def write(self, filename=None, tabs=0): + if self.out_handle: + raise Exception("Already writing file") + + if filename is None: + filename = self.out_filename + + self.out_handle = open(filename, 'wb') + + self.writeHeaders() + + self.writeOfx(tabs=tabs) + + self.out_handle.flush() + self.out_handle.close() + self.out_handle = None diff --git a/ofxparse/ofxutil.py b/ofxparse/ofxutil.py index 17c2f36..0002ee1 100644 --- a/ofxparse/ofxutil.py +++ b/ofxparse/ofxutil.py @@ -12,9 +12,11 @@ else: import six + class InvalidOFXStructureException(Exception): pass + class OfxData(object): def __init__(self, tag): self.nodes = odict.OrderedDict() @@ -38,7 +40,8 @@ class OfxData(object): del self.nodes[name] def __setattr__(self, name, value): - if name in self.__dict__ or name in ['nodes', 'tag', 'data', 'headers', 'xml']: + if name in self.__dict__ or name in ['nodes', 'tag', 'data', '\ + headers', 'xml']: self.__dict__[name] = value else: self.del_tag(name) @@ -100,12 +103,14 @@ class OfxData(object): return len(self.nodes) def __str__(self): - return os.linesep.join("\t" * line[1] + line[0] for line in self.format()) + return os.linesep.join("\t" * line[1] + line[0] for line \ + in self.format()) def format(self): if self.data or not self.nodes: if self.tag.upper() == "OFX": - return [["<%s>%s" % (self.tag, self.data if self.data else "", self.tag), 0]] + return [["<%s>%s" % (self.tag, self.data \ + if self.data else "", self.tag), 0]] return [["<%s>%s" % (self.tag, self.data), 0]] else: ret = [["<%s>" % self.tag, -1]] @@ -129,10 +134,12 @@ class OfxUtil(OfxData): self.headers = odict.OrderedDict() self.xml = "" if ofx_data: - if isinstance(ofx_data, six.string_types) and not ofx_data.lower().endswith('.ofx'): + if isinstance(ofx_data, six.string_types) and not \ + ofx_data.lower().endswith('.ofx'): self.parse(ofx_data) else: - self.parse(open(ofx_data).read() if isinstance(ofx_data, six.string_types) else ofx_data.read()) + self.parse(open(ofx_data).read() if isinstance(\ + ofx_data, six.string_types) else ofx_data.read()) def parse(self, ofx): try: @@ -168,24 +175,27 @@ class OfxUtil(OfxData): for i, tag in enumerate(tags): gt = tag.index(">") if tag[1] != "/": - #Is an opening tag + # Is an opening tag if not can_open: - tags[i - 1] = tags[i - 1] + "" + tags[i - 1] = tags[i - 1] + "" can_open = True tag_name = tag[1:gt].split()[0] heirarchy.append(tag_name) if len(tag) > gt + 1: can_open = False else: - #Is a closing tag + # Is a closing tag tag_name = tag[2:gt].split()[0] if tag_name not in heirarchy: - #Close tag with no matching open, so delete it + # Close tag with no matching open, so delete it tags[i] = tag[gt + 1:] else: - #Close tag with matching open, but other open tags that need to be closed first + # Close tag with matching open, but other open + # tags that need to be closed first while(tag_name != heirarchy[-1]): - tags[i - 1] = tags[i - 1] + "" + tags[i - 1] = tags[i - 1] + "" can_open = True heirarchy.pop() @@ -209,7 +219,8 @@ class OfxUtil(OfxData): f.write(str(self)) def __str__(self): - ret = os.linesep.join(":".join(line) for line in six.iteritems(self.headers)) + os.linesep * 2 + ret = os.linesep.join(":".join(line) for line in \ + six.iteritems(self.headers)) + os.linesep * 2 ret += super(OfxUtil, self).__str__() return ret @@ -219,22 +230,20 @@ if __name__ == "__main__": ofx = OfxUtil(fixtures + 'checking.ofx') # ofx = OfxUtil(fixtures + 'fidelity.ofx') - - #Manipulate OFX file via XML library + # Manipulate OFX file via XML library # for transaction in ofx.xml.iter('STMTTRN'): # transaction.find('NAME').text = transaction.find('MEMO').text # transaction.remove(transaction.find('MEMO')) # ofx.reload_xml() - - #Manipulate OFX file via object tree built from XML + # Manipulate OFX file via object tree built from XML # for transaction in ofx.bankmsgsrsv1.stmttrnrs.stmtrs.banktranlist.stmttrn: # transaction.name = transaction.memo # del transaction.memo # transaction.notes = "Acknowledged" - - #Modified sytnax for object tree data manipulation - #I'm using the __getitem__ method like the xml.iter method from ElementTree, as a recursive search + # Modified sytnax for object tree data manipulation + # I'm using the __getitem__ method like the xml.iter method from + # ElementTree, as a recursive search for transaction in ofx['stmttrn']: transaction.name = transaction.memo del transaction.memo @@ -248,7 +257,7 @@ if __name__ == "__main__": # print(ofx) - #Write OFX data to output file + # Write OFX data to output file # ofx.write('out.ofx') # for file_name in os.listdir(fixtures): diff --git a/setup.py b/setup.py index 8450a12..7ef8b6c 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,6 @@ + +import codecs +import os import re import sys @@ -10,30 +13,37 @@ VERSION = re.search(r"__version__ = '(.*?)'", open("ofxparse/__init__.py").read()).group(1) # Use BeautifulSoup 3 on Python 2.5 and earlier and BeautifulSoup 4 otherwise -if sys.version_info < (2,6): +if sys.version_info < (2, 6): REQUIRES = [ "beautifulSoup>=3.0", ] else: REQUIRES = [ - "beautifulsoup4" + "beautifulsoup4", + "lxml", ] -if sys.version_info < (2,7): +if sys.version_info < (2, 7): REQUIRES.extend([ "ordereddict>=1.1", ]) REQUIRES.extend([ 'six', + 'lxml' ]) +README = os.path.join(os.path.dirname(__file__), 'README.rst') + +with codecs.open(README, encoding='utf8') as f: + LONG_DESCRIPTION = f.read() + setup_params = dict( name='ofxparse', version=VERSION, description=("Tools for working with the OFX (Open Financial Exchange)" " file format"), - long_description=open("./README", "r").read(), + long_description=LONG_DESCRIPTION, # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ "Development Status :: 4 - Beta", diff --git a/tests/fixtures/investment_401k.ofx b/tests/fixtures/investment_401k.ofx new file mode 100644 index 0000000..1fc0898 --- /dev/null +++ b/tests/fixtures/investment_401k.ofx @@ -0,0 +1,180 @@ +OFXHEADER:100 +DATA:OFXSGML +VERSION:102 +SECURITY:NONE +ENCODING:USASCII +CHARSET:1252 +COMPRESSION:NONE +OLDFILEUID:NONE +NEWFILEUID:NONE + + + + + + 0 + INFO + SUCCESS + + 20150909084609.717[-6:MDT] + ENG + + EXAMPLE + 1234 + + 1234 + + + + + 0 + + 0 + INFO + SUCCESS + + + 20140630000000.000[-6:MDT] + USD + + example.org + 12345678.123456-01 + + + 20140401000000.000[-6:MDT] + 20140630000000.000[-6:MDT] + + + + 1 + 20140617000000.000[-6:MDT] + + + FOO + PRIVATE + + 8.846699 + 22.2908 + -197.2 + OTHER + OTHER + + BUY + + + + 2 + 20140630000000.000[-6:MDT] + + + BAR + PRIVATE + + OTHER + 6.800992 + IN + LONG + 29.214856 + + + + 3 + 20140630000000.000[-6:MDT] + + + BAZ + PRIVATE + + OTHER + -9.060702 + OUT + LONG + 21.928764 + + + + + + + FOO + PRIVATE + + CASH + LONG + 17.604312 + 22.517211 + 396.4 + 20140630000000.000[-6:MDT] + + + + + + BAR + PRIVATE + + CASH + LONG + 13.550983 + 29.214855 + 395.89 + 20140630000000.000[-6:MDT] + + + + + + BAZ + PRIVATE + + CASH + LONG + 0.0 + 0.0 + 0.0 + 20140630000000.000[-6:MDT] + + + + + 1000.00 + + + + + + + + + + BAR + PRIVATE + + BAR Index Fund + BAR + + + + + + FOO + PRIVATE + + Foo Index Fund + FOO + + + + + + BAZ + PRIVATE + + Baz Fund + BAZ + + + + + + diff --git a/tests/fixtures/suncorp.ofx b/tests/fixtures/suncorp.ofx new file mode 100644 index 0000000..4a0558d --- /dev/null +++ b/tests/fixtures/suncorp.ofx @@ -0,0 +1,56 @@ + + + + + + + 0 + INFO + + 20131215 + ENG + + SUNCORP + 484-799 + + + + + + 1 + + 0 + INFO + + + AUD + + SUNCORP + 123456789 + CHECKING + + + 20130618 + 20131215 + + DEBIT + 20131215 + -16.85 + 1 + 0 + + + + + + 1234.12 + 20131215 + + + 1234.12 + 20131215 + + + + + \ No newline at end of file diff --git a/tests/fixtures/vanguard401k.ofx b/tests/fixtures/vanguard401k.ofx new file mode 100644 index 0000000..0443ca1 --- /dev/null +++ b/tests/fixtures/vanguard401k.ofx @@ -0,0 +1,11 @@ +OFXHEADER:100 +DATA:OFXSGML +VERSION:102 +SECURITY:NONE +ENCODING:USASCII +CHARSET:1252 +COMPRESSION:NONE +OLDFILEUID:NONE +NEWFILEUID:NONE + +0INFOSuccessful Sign On20141018150740[-5:EST]ENG20140605083000Vanguard84022foo84022USER34500INFO20141017160000.000[-5:EST]USDvanguard.com012345620140916160000.000[-5:EST]20141018150740.000[-5:EST]1234567890123456790AAA20140926160000.000[-5:EST]20140926160000.000[-5:EST]Price as of date based on closing price92202V351CUSIP14.6113746.06-673.0CASHOTHERPRETAXBUY1234567890123456791AAA20140926160000.000[-5:EST]20140926160000.000[-5:EST]Price as of date based on closing price92202V351CUSIP7.3056846.06-336.5CASHOTHERMATCHBUY1234567890123456793AAA20141010160000.000[-5:EST]20141010160000.000[-5:EST]Price as of date based on closing price92202V351CUSIP15.2503944.13-673.0CASHOTHERPRETAXBUY1234567890123456794AAA20141010160000.000[-5:EST]20141010160000.000[-5:EST]Price as of date based on closing price92202V351CUSIP7.6251944.13-336.5CASHOTHERMATCHBUY1234567890123456795AAA20130905160000.000[-5:EST]20130906160000.000[-5:EST]Investment Expense92202V351CUSIPCASH-0.04241OUTLONG39.37MATCH92202V351CUSIPOTHERLONG117.50644.015171.4420141017160000.000[-5:EST]Price as of date based on closing priceOTHERNONVESTYYGOOGLE INC. 401(K) SAVINGS PLAN100.00.00.00.00.00.00.00.00.00.092202V351CUSIPTarget Retirement 2050 Trust Plus165944.0120141017160000.000[-5:EST]Price as of date based on closing price diff --git a/tests/test_parse.py b/tests/test_parse.py index 35260ce..476b28e 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -1,17 +1,18 @@ from __future__ import absolute_import -from ofxparse.ofxparse import soup_maker +import os from datetime import datetime, timedelta from decimal import Decimal from unittest import TestCase import sys -sys.path.append('..') +sys.path.insert(0, os.path.abspath('..')) import six from .support import open_file from ofxparse import OfxParser, AccountType, Account, Statement, Transaction -from ofxparse.ofxparse import OfxFile, OfxPreprocessedFile, OfxParserException +from ofxparse.ofxparse import OfxFile, OfxPreprocessedFile, OfxParserException, soup_maker + class TestOfxPreprocessedFile(TestCase): @@ -41,8 +42,8 @@ NEWFILEUID:NONE abNet2222Gross3333 """ ofx_file = OfxPreprocessedFile(fh) - data = ofx_file.fh.read() - self.assertEqual(data,expect) + data = ofx_file.fh.read() + self.assertEqual(data, expect) def testHeaders(self): expect = {"OFXHEADER": six.u("100"), @@ -124,7 +125,6 @@ NEWFILEUID:NONE self.assertEquals(len(ofx_file.headers.keys()), 2) - class TestOfxFile(TestCase): def testHeaders(self): expect = {"OFXHEADER": six.u("100"), @@ -219,7 +219,7 @@ class TestParse(TestCase): def testThatParseFailsIfAPathIsPassedIn(self): # A file handle should be passed in, not the path. - self.assertRaises(RuntimeError, OfxParser.parse, '/foo/bar') + self.assertRaises(TypeError, OfxParser.parse, '/foo/bar') def testThatParseReturnsAResultWithABankAccount(self): ofx = OfxParser.parse(open_file('bank_medium.ofx')) @@ -232,11 +232,15 @@ class TestParse(TestCase): self.assertEquals('00', ofx.account.branch_id) self.assertEquals('CHECKING', ofx.account.account_type) self.assertEquals(Decimal('382.34'), ofx.account.statement.balance) + self.assertEquals(datetime(2009, 5, 23, 12, 20, 17), + ofx.account.statement.balance_date) # Todo: support values in decimal or int form. # self.assertEquals('15', # ofx.bank_account.statement.balance_in_pennies) self.assertEquals( Decimal('682.34'), ofx.account.statement.available_balance) + self.assertEquals(datetime(2009, 5, 23, 12, 20, 17), + ofx.account.statement.available_balance_date) self.assertEquals( datetime(2009, 4, 1), ofx.account.statement.start_date) self.assertEquals( @@ -271,10 +275,13 @@ class TestStringToDate(TestCase): self.assertRaises(ValueError, OfxParser.parseOfxDateTime, bad_string) bad_but_close_string = '881103' - self.assertRaises(ValueError, OfxParser.parseOfxDateTime, bad_string) + self.assertRaises(ValueError, OfxParser.parseOfxDateTime, bad_but_close_string) no_month_string = '19881301' - self.assertRaises(ValueError, OfxParser.parseOfxDateTime, bad_string) + self.assertRaises(ValueError, OfxParser.parseOfxDateTime, no_month_string) + + def test_returns_none(self): + self.assertIsNone(OfxParser.parseOfxDateTime('00000000')) def test_parses_correct_time(self): '''Test whether it can parse correct time for some valid time fields''' @@ -387,8 +394,58 @@ class TestParseStatement(TestCase): datetime(2009, 5, 23, 12, 20, 17), statement.end_date) self.assertEquals(1, len(statement.transactions)) self.assertEquals(Decimal('382.34'), statement.balance) + self.assertEquals(datetime(2009, 5, 23, 12, 20, 17), statement.balance_date) self.assertEquals(Decimal('682.34'), statement.available_balance) + self.assertEquals(datetime(2009, 5, 23, 12, 20, 17), statement.available_balance_date) + def testThatParseStatementWithBlankDatesReturnsAStatement(self): + input = ''' + + 20090523122017 + + 0 + INFO + OK + + + CAD + + 160000100 + 12300 000012345678 + CHECKING + + + 00000000 + 00000000 + + POS + 20090401122017.000[-5:EST] + -6.60 + 0000123456782009040100001 + MCDONALD'S #112 + POS MERCHANDISE;MCDONALD'S #112 + + + + 382.34 + 20090523122017 + + + 682.34 + 20090523122017 + + + + ''' + txn = soup_maker(input) + statement = OfxParser.parseStatement(txn.find('stmttrnrs')) + self.assertEquals(None, statement.start_date) + self.assertEquals(None, statement.end_date) + self.assertEquals(1, len(statement.transactions)) + self.assertEquals(Decimal('382.34'), statement.balance) + self.assertEquals(datetime(2009, 5, 23, 12, 20, 17), statement.balance_date) + self.assertEquals(Decimal('682.34'), statement.available_balance) + self.assertEquals(datetime(2009, 5, 23, 12, 20, 17), statement.available_balance_date) class TestStatement(TestCase): def testThatANewStatementIsValid(self): @@ -420,7 +477,6 @@ class TestParseTransaction(TestCase): self.assertEquals("MCDONALD'S #112", transaction.payee) self.assertEquals("POS MERCHANDISE;MCDONALD'S #112", transaction.memo) - def testThatParseTransactionWithFieldCheckNum(self): input = ''' @@ -436,6 +492,61 @@ class TestParseTransaction(TestCase): transaction = OfxParser.parseTransaction(txn.find('stmttrn')) self.assertEquals('700', transaction.checknum) + def testThatParseTransactionWithCommaAsDecimalPoint(self): + input = ''' + + POS + 20090401122017.000[-5:EST] + -1006,60 + 0000123456782009040100001 + MCDONALD'S #112 + POS MERCHANDISE;MCDONALD'S #112 + +''' + txn = soup_maker(input) + transaction = OfxParser.parseTransaction(txn.find('stmttrn')) + self.assertEquals(Decimal('-1006.60'), transaction.amount) + + def testThatParseTransactionWithCommaAsDecimalPointAndDotAsSeparator(self): + input = ''' + + POS + 20090401122017.000[-5:EST] + -1.006,60 + 0000123456782009040100001 + MCDONALD'S #112 + POS MERCHANDISE;MCDONALD'S #112 + +''' + txn = soup_maker(input) + with self.assertRaises(OfxParserException): + transaction = OfxParser.parseTransaction(txn.find('stmttrn')) + + def testThatParseTransactionWithNullAmountIgnored(self): + """A null transaction value is converted to 0. + + Some banks use a null transaction to include interest + rate changes on statements. + """ + input_template = ''' + + DEP + 20130306 + {amount} + 2013030601009100 + 700 + DEPOSITO ONLINE + +''' + for amount in ("null", "-null"): + input = input_template.format(amount=amount) + txn = soup_maker(input) + + transaction = OfxParser.parseTransaction(txn.find('stmttrn')) + + self.assertEquals(0, transaction.amount) + + class TestTransaction(TestCase): def testThatAnEmptyTransactionIsValid(self): t = Transaction() @@ -473,7 +584,6 @@ class TestInvestmentAccount(TestCase): # Success! - class TestVanguardInvestmentStatement(TestCase): def testForUnclosedTags(self): ofx = OfxParser.parse(open_file('vanguard.ofx')) @@ -497,6 +607,20 @@ class TestVanguardInvestmentStatement(TestCase): self.assertEquals(len(ofx.security_list), 2) +class TestVanguard401kStatement(TestCase): + def testReadTransfer(self): + ofx = OfxParser.parse(open_file('vanguard401k.ofx')) + self.assertTrue(hasattr(ofx, 'account')) + self.assertTrue(hasattr(ofx.account, 'statement')) + self.assertTrue(hasattr(ofx.account.statement, 'transactions')) + self.assertEquals(len(ofx.account.statement.transactions), 5) + self.assertEquals(ofx.account.statement.transactions[-1].id, + '1234567890123456795AAA') + self.assertEquals('transfer', ofx.account.statement.transactions[-1].type) + self.assertEquals(ofx.account.statement.transactions[-1].inv401ksource, + 'MATCH') + + class TestFidelityInvestmentStatement(TestCase): def testForUnclosedTags(self): ofx = OfxParser.parse(open_file('fidelity.ofx')) @@ -510,6 +634,66 @@ class TestFidelityInvestmentStatement(TestCase): self.assertEquals(len(ofx.security_list), 7) +class Test401InvestmentStatement(TestCase): + def testTransferAggregate(self): + ofx = OfxParser.parse(open_file('investment_401k.ofx')) + expected_txns = [{'id': '1', + 'type': 'buymf', + 'units': Decimal('8.846699'), + 'unit_price': Decimal('22.2908'), + 'total': Decimal('-197.2'), + 'security': 'FOO'}, + {'id': '2', + 'type': 'transfer', + 'units': Decimal('6.800992'), + 'unit_price': Decimal('29.214856'), + 'total': Decimal('0.0'), + 'security': 'BAR'}, + {'id': '3', + 'type': 'transfer', + 'units': Decimal('-9.060702'), + 'unit_price': Decimal('21.928764'), + 'total': Decimal('0.0'), + 'security': 'BAZ'}] + for txn, expected_txn in zip(ofx.account.statement.transactions, expected_txns): + self.assertEquals(txn.id, expected_txn['id']) + self.assertEquals(txn.type, expected_txn['type']) + self.assertEquals(txn.units, expected_txn['units']) + self.assertEquals(txn.unit_price, expected_txn['unit_price']) + self.assertEquals(txn.total, expected_txn['total']) + self.assertEquals(txn.security, expected_txn['security']) + + expected_positions = [{'security': 'FOO', + 'units': Decimal('17.604312'), + 'unit_price': Decimal('22.517211')}, + {'security': 'BAR', + 'units': Decimal('13.550983'), + 'unit_price': Decimal('29.214855')}, + {'security': 'BAZ', + 'units': Decimal('0.0'), + 'unit_price': Decimal('0.0')}] + for pos, expected_pos in zip(ofx.account.statement.positions, expected_positions): + self.assertEquals(pos.security, expected_pos['security']) + self.assertEquals(pos.units, expected_pos['units']) + self.assertEquals(pos.unit_price, expected_pos['unit_price']) + + +class TestSuncorpBankStatement(TestCase): + def testCDATATransactions(self): + ofx = OfxParser.parse(open_file('suncorp.ofx')) + accounts = ofx.accounts + self.assertEquals(len(accounts), 1) + account = accounts[0] + transactions = account.statement.transactions + self.assertEquals(len(transactions), 1) + transaction = transactions[0] + self.assertEquals(transaction.payee, "EFTPOS WDL HANDYWAY ALDI STORE") + self.assertEquals( + transaction.memo, + "EFTPOS WDL HANDYWAY ALDI STORE GEELONG WEST VICAU") + self.assertEquals(transaction.amount, Decimal("-16.85")) + + class TestAccountInfoAggregation(TestCase): def testForFourAccounts(self): ofx = OfxParser.parse(open_file('account_listing_aggregation.ofx')) @@ -596,6 +780,7 @@ class TestGracefulFailures(TestCase): self.assertRaises(OfxParserException, OfxParser.parse, open_file('fail_nice/empty_balance.ofx')) + class TestParseSonrs(TestCase): def testSuccess(self): diff --git a/tests/test_write.py b/tests/test_write.py index 3366217..28361b0 100644 --- a/tests/test_write.py +++ b/tests/test_write.py @@ -6,6 +6,7 @@ import sys sys.path.append('..') from .support import open_file + class TestOfxWrite(TestCase): def test_write(self): test_file = open_file('fidelity.ofx') -- cgit v1.2.3