diff options
-rw-r--r-- | AUTHORS | 9 | ||||
-rw-r--r-- | PKG-INFO | 48 | ||||
-rw-r--r-- | README.rst (renamed from README) | 46 | ||||
-rw-r--r-- | ofxparse.egg-info/PKG-INFO | 48 | ||||
-rw-r--r-- | ofxparse.egg-info/SOURCES.txt | 6 | ||||
-rw-r--r-- | ofxparse.egg-info/requires.txt | 4 | ||||
-rw-r--r-- | ofxparse/__init__.py | 3 | ||||
-rw-r--r-- | ofxparse/ofxparse.py | 337 | ||||
-rw-r--r-- | ofxparse/ofxprinter.py | 194 | ||||
-rw-r--r-- | ofxparse/ofxutil.py | 49 | ||||
-rw-r--r-- | setup.py | 18 | ||||
-rw-r--r-- | tests/fixtures/investment_401k.ofx | 180 | ||||
-rw-r--r-- | tests/fixtures/suncorp.ofx | 56 | ||||
-rw-r--r-- | tests/fixtures/vanguard401k.ofx | 11 | ||||
-rw-r--r-- | tests/test_parse.py | 207 | ||||
-rw-r--r-- | tests/test_write.py | 1 |
16 files changed, 1057 insertions, 160 deletions
@@ -14,3 +14,12 @@ Andre Smolik greggles@github mikeivanov@github danc86@github +Erik Hetzner +Panagiotis Issaris +Brett Trotter +Joe Cabrera +Michael Nelson +Nathan Grigg +Wes Turner +Ehud Ben-Reuven +Joseph Walton @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: ofxparse -Version: 0.14 +Version: 0.15 Summary: Tools for working with the OFX (Open Financial Exchange) file format Home-page: http://sites.google.com/site/ofxparse Author: Jerry Seutter @@ -23,11 +23,13 @@ Description: ofxparse Example Usage ============= - Here's a sample program:: + Here's a sample program - from ofxparse import OfxParser + .. code:: python - ofx = OfxParser.parse(file('file.ofx')) + from ofxparse import OfxParser + with codecs.open('file.ofx') as fileobj: + ofx = OfxParser.parse(fileobj) ofx.accounts # An account with information ofx.account.number # The account number ofx.account.routing_number # The transit id (sometimes called branch number) @@ -41,7 +43,8 @@ Description: ofxparse Help! ===== - Sample .ofx files are very useful. If you want to help us out, please edit + Sample ``.ofx`` and ``.qfx`` files are very useful. + If you want to help us out, please edit all identifying information from the file and then email it to jseutter dot ofxparse at gmail dot com. @@ -49,25 +52,46 @@ Description: ofxparse =========== Prerequisites:: - (Ubuntu) sudo apt-get install python-beautifulsoup python-nose python-coverage-test-runner - (pip) pip install BeautifulSoup nose coverage + # Ubuntu + sudo apt-get install python-beautifulsoup python-nose python-coverage-test-runner + + # pip for Python 3: + pip install BeautifulSoup4 six lxml nose coverage + + # pip for Python 2: + pip install BeautifulSoup six nose coverage + + Tests: + Simply running the ``nosetests`` command should run the tests. - Tests:: - Simply running the "nose" command should run the tests. If you don't have nose - installed, the following might also work: + .. code:: bash + + nosetests + + If you don't have nose installed, the following might also work: + + .. code:: bash python -m unittest tests.test_parse - Test Coverage Report:: + Test Coverage Report: + + .. code:: bash coverage run -m unittest tests.test_parse + + # text report + coverage report + + # html report coverage html firefox htmlcov/index.html Homepage ======== - http://sites.google.com/site/ofxparse + | Homepage: https://sites.google.com/site/ofxparse + | Source: https://github.com/jseutter/ofxparse License ======= @@ -15,11 +15,13 @@ improved. See the Help! section below for directions on how to do this. Example Usage ============= -Here's a sample program:: +Here's a sample program - from ofxparse import OfxParser +.. code:: python - ofx = OfxParser.parse(file('file.ofx')) + from ofxparse import OfxParser + with codecs.open('file.ofx') as fileobj: + ofx = OfxParser.parse(fileobj) ofx.accounts # An account with information ofx.account.number # The account number ofx.account.routing_number # The transit id (sometimes called branch number) @@ -33,7 +35,8 @@ Here's a sample program:: Help! ===== -Sample .ofx files are very useful. If you want to help us out, please edit +Sample ``.ofx`` and ``.qfx`` files are very useful. +If you want to help us out, please edit all identifying information from the file and then email it to jseutter dot ofxparse at gmail dot com. @@ -41,25 +44,46 @@ Development =========== Prerequisites:: -(Ubuntu) sudo apt-get install python-beautifulsoup python-nose python-coverage-test-runner -(pip) pip install BeautifulSoup nose coverage + # Ubuntu + sudo apt-get install python-beautifulsoup python-nose python-coverage-test-runner + + # pip for Python 3: + pip install BeautifulSoup4 six lxml nose coverage + + # pip for Python 2: + pip install BeautifulSoup six nose coverage + +Tests: +Simply running the ``nosetests`` command should run the tests. -Tests:: -Simply running the "nose" command should run the tests. If you don't have nose -installed, the following might also work: +.. code:: bash + + nosetests + +If you don't have nose installed, the following might also work: + +.. code:: bash python -m unittest tests.test_parse -Test Coverage Report:: +Test Coverage Report: + +.. code:: bash coverage run -m unittest tests.test_parse + + # text report + coverage report + + # html report coverage html firefox htmlcov/index.html Homepage ======== -http://sites.google.com/site/ofxparse +| Homepage: https://sites.google.com/site/ofxparse +| Source: https://github.com/jseutter/ofxparse License ======= diff --git a/ofxparse.egg-info/PKG-INFO b/ofxparse.egg-info/PKG-INFO index 83be6d8..c19dfd2 100644 --- a/ofxparse.egg-info/PKG-INFO +++ b/ofxparse.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: ofxparse -Version: 0.14 +Version: 0.15 Summary: Tools for working with the OFX (Open Financial Exchange) file format Home-page: http://sites.google.com/site/ofxparse Author: Jerry Seutter @@ -23,11 +23,13 @@ Description: ofxparse Example Usage ============= - Here's a sample program:: + Here's a sample program - from ofxparse import OfxParser + .. code:: python - ofx = OfxParser.parse(file('file.ofx')) + from ofxparse import OfxParser + with codecs.open('file.ofx') as fileobj: + ofx = OfxParser.parse(fileobj) ofx.accounts # An account with information ofx.account.number # The account number ofx.account.routing_number # The transit id (sometimes called branch number) @@ -41,7 +43,8 @@ Description: ofxparse Help! ===== - Sample .ofx files are very useful. If you want to help us out, please edit + Sample ``.ofx`` and ``.qfx`` files are very useful. + If you want to help us out, please edit all identifying information from the file and then email it to jseutter dot ofxparse at gmail dot com. @@ -49,25 +52,46 @@ Description: ofxparse =========== Prerequisites:: - (Ubuntu) sudo apt-get install python-beautifulsoup python-nose python-coverage-test-runner - (pip) pip install BeautifulSoup nose coverage + # Ubuntu + sudo apt-get install python-beautifulsoup python-nose python-coverage-test-runner + + # pip for Python 3: + pip install BeautifulSoup4 six lxml nose coverage + + # pip for Python 2: + pip install BeautifulSoup six nose coverage + + Tests: + Simply running the ``nosetests`` command should run the tests. - Tests:: - Simply running the "nose" command should run the tests. If you don't have nose - installed, the following might also work: + .. code:: bash + + nosetests + + If you don't have nose installed, the following might also work: + + .. code:: bash python -m unittest tests.test_parse - Test Coverage Report:: + Test Coverage Report: + + .. code:: bash coverage run -m unittest tests.test_parse + + # text report + coverage report + + # html report coverage html firefox htmlcov/index.html Homepage ======== - http://sites.google.com/site/ofxparse + | Homepage: https://sites.google.com/site/ofxparse + | Source: https://github.com/jseutter/ofxparse License ======= diff --git a/ofxparse.egg-info/SOURCES.txt b/ofxparse.egg-info/SOURCES.txt index 6cff2c7..21fc7ff 100644 --- a/ofxparse.egg-info/SOURCES.txt +++ b/ofxparse.egg-info/SOURCES.txt @@ -1,12 +1,13 @@ AUTHORS LICENSE MANIFEST.in -README +README.rst setup.cfg setup.py ofxparse/__init__.py ofxparse/mcc.py ofxparse/ofxparse.py +ofxparse/ofxprinter.py ofxparse/ofxutil.py ofxparse.egg-info/PKG-INFO ofxparse.egg-info/SOURCES.txt @@ -24,13 +25,16 @@ tests/fixtures/bank_medium.ofx tests/fixtures/bank_small.ofx tests/fixtures/checking.ofx tests/fixtures/fidelity.ofx +tests/fixtures/investment_401k.ofx tests/fixtures/investment_medium.ofx tests/fixtures/multiple_accounts.ofx tests/fixtures/multiple_accounts2.ofx tests/fixtures/signon_fail.ofx tests/fixtures/signon_success.ofx tests/fixtures/signon_success_no_message.ofx +tests/fixtures/suncorp.ofx tests/fixtures/vanguard.ofx +tests/fixtures/vanguard401k.ofx tests/fixtures/fail_nice/date_missing.ofx tests/fixtures/fail_nice/decimal_error.ofx tests/fixtures/fail_nice/empty_balance.ofx
\ No newline at end of file diff --git a/ofxparse.egg-info/requires.txt b/ofxparse.egg-info/requires.txt index d33d9e5..c038268 100644 --- a/ofxparse.egg-info/requires.txt +++ b/ofxparse.egg-info/requires.txt @@ -1,2 +1,4 @@ beautifulsoup4 -six
\ No newline at end of file +lxml +six +lxml diff --git a/ofxparse/__init__.py b/ofxparse/__init__.py index f1fb25b..a08ba77 100644 --- a/ofxparse/__init__.py +++ b/ofxparse/__init__.py @@ -1,5 +1,6 @@ from __future__ import absolute_import from .ofxparse import OfxParser, AccountType, Account, Statement, Transaction +from .ofxprinter import OfxPrinter -__version__ = '0.14' +__version__ = '0.15' diff --git a/ofxparse/ofxparse.py b/ofxparse/ofxparse.py index a66df88..5686030 100644 --- a/ofxparse/ofxparse.py +++ b/ofxparse/ofxparse.py @@ -22,13 +22,37 @@ else: from . import mcc + +def skip_headers(fh): + ''' + Prepare `fh` for parsing by BeautifulSoup by skipping its OFX + headers. + ''' + if fh is None or isinstance(fh, six.string_types): + return + fh.seek(0) + header_re = re.compile(r"^\s*\w+:\s*\w+\s*$") + while True: + pos = fh.tell() + line = fh.readline() + if not line: + break + if header_re.search(line) is None: + fh.seek(pos) + return + + def soup_maker(fh): + skip_headers(fh) try: from bs4 import BeautifulSoup - return BeautifulSoup(fh) + soup = BeautifulSoup(fh, "xml") + for tag in soup.findAll(): + tag.name = tag.name.lower() except ImportError: from BeautifulSoup import BeautifulStoneSoup - return BeautifulStoneSoup(fh) + soup = BeautifulStoneSoup(fh) + return soup def try_decode(string, encoding): @@ -36,11 +60,13 @@ def try_decode(string, encoding): string = string.decode(encoding) return string + def is_iterable(candidate): - if sys.version_info < (2,6): + if sys.version_info < (2, 6): return hasattr(candidate, 'next') return isinstance(candidate, collections.Iterable) + @contextlib.contextmanager def save_pos(fh): """ @@ -54,6 +80,7 @@ def save_pos(fh): finally: fh.seek(orig_pos) + class OfxFile(object): def __init__(self, fh): """ @@ -76,7 +103,7 @@ class OfxFile(object): head_data = self.fh.read(1024 * 10) head_data = head_data[:head_data.find(six.b('<'))] - for line in re.split(six.b('\r?\n?'), head_data): + for line in head_data.splitlines(): # Newline? if line.strip() == six.b(""): break @@ -138,7 +165,7 @@ class OfxFile(object): class OfxPreprocessedFile(OfxFile): def __init__(self, fh): - super(OfxPreprocessedFile,self).__init__(fh) + super(OfxPreprocessedFile, self).__init__(fh) if self.fh is None: return @@ -146,19 +173,21 @@ class OfxPreprocessedFile(OfxFile): ofx_string = self.fh.read() # find all closing tags as hints - closing_tags = [ t.upper() for t in re.findall(r'(?i)</([a-z0-9_\.]+)>', ofx_string) ] + closing_tags = [t.upper() for t in re.findall(r'(?i)</([a-z0-9_\.]+)>', + ofx_string)] # close all tags that don't have closing tags and # leave all other data intact last_open_tag = None - tokens = re.split(r'(?i)(</?[a-z0-9_\.]+>)', ofx_string) - new_fh = StringIO() - for idx,token in enumerate(tokens): + tokens = re.split(r'(?i)(</?[a-z0-9_\.]+>)', ofx_string) + new_fh = StringIO() + for idx, token in enumerate(tokens): is_closing_tag = token.startswith('</') is_processing_tag = token.startswith('<?') is_cdata = token.startswith('<!') is_tag = token.startswith('<') and not is_cdata - is_open_tag = is_tag and not is_closing_tag and not is_processing_tag + is_open_tag = is_tag and not is_closing_tag \ + and not is_processing_tag if is_tag: if last_open_tag is not None: new_fh.write("</%s>" % last_open_tag) @@ -175,7 +204,8 @@ class OfxPreprocessedFile(OfxFile): class Ofx(object): def __str__(self): return "" -# headers = "\r\n".join(":".join(el if el else "NONE" for el in item) for item in six.iteritems(self.headers)) +# headers = "\r\n".join(":".join(el if el else "NONE" for el in item) +# for item in six.iteritems(self.headers)) # headers += "\r\n\r\n" # # return headers + str(self.signon) @@ -187,6 +217,7 @@ class AccountType(object): class Account(object): def __init__(self): + self.curdef = None self.statement = None self.account_id = '' self.routing_number = '' @@ -216,25 +247,52 @@ class Security: self.ticker = ticker self.memo = memo + class Signon: - def __init__(self, code, severity, message): - self.code = code - self.severity = severity - self.message = message - if int(code) == 0: + def __init__(self, keys): + self.code = keys['code'] + self.severity = keys['severity'] + self.message = keys['message'] + self.dtserver = keys['dtserver'] + self.language = keys['language'] + self.dtprofup = keys['dtprofup'] + self.fi_org = keys['org'] + self.fi_fid = keys['fid'] + self.intu_bid = keys['intu.bid'] + + if int(self.code) == 0: self.success = True else: self.success = False def __str__(self): - ret = "\t<SIGNONMSGSRSV1>\r\n" + "\t\t<SONRS>\r\n" + "\t\t\t<STATUS>\r\n" + ret = "\t<SIGNONMSGSRSV1>\r\n" + "\t\t<SONRS>\r\n" + \ + "\t\t\t<STATUS>\r\n" ret += "\t\t\t\t<CODE>%s\r\n" % self.code ret += "\t\t\t\t<SEVERITY>%s\r\n" % self.severity if self.message: ret += "\t\t\t\t<MESSAGE>%s\r\n" % self.message - ret += "\t\t\t</STATUS>\r\n" + "\t\t</SONRS>\r\n" + "\t</SIGNONMSGSRSV1>\r\n" + ret += "\t\t\t</STATUS>\r\n" + if self.dtserver is not None: + ret += "\t\t\t<DTSERVER>" + self.dtserver + "\r\n" + if self.language is not None: + ret += "\t\t\t<LANGUAGE>" + self.language + "\r\n" + if self.dtprofup is not None: + ret += "\t\t\t<DTPROFUP>" + self.dtprofup + "\r\n" + if (self.fi_org is not None) or (self.fi_fid is not None): + ret += "\t\t\t<FI>\r\n" + if self.fi_org is not None: + ret += "\t\t\t\t<ORG>" + self.fi_org + "\r\n" + if self.fi_fid is not None: + ret += "\t\t\t\t<FID>" + self.fi_fid + "\r\n" + ret += "\t\t\t</FI>\r\n" + if self.intu_bid is not None: + ret += "\t\t\t<INTU.BID>" + self.intu_bid + "\r\n" + ret += "\t\t</SONRS>\r\n" + ret += "\t</SIGNONMSGSRSV1>\r\n" return ret + class Statement(object): def __init__(self): self.start_date = '' @@ -272,20 +330,29 @@ class Transaction(object): class InvestmentTransaction(object): - (Unknown, BuyMF, SellMF, Reinvest, BuyStock, SellStock) = [x for x in range(-1, 5)] + AGGREGATE_TYPES = ['buydebt', 'buymf', 'buyopt', 'buyother', + 'buystock', 'closureopt', 'income', + 'invexpense', 'jrnlfund', 'jrnlsec', + 'margininterest', 'reinvest', 'retofcap', + 'selldebt', 'sellmf', 'sellopt', 'sellother', + 'sellstock', 'split', 'transfer'] + def __init__(self, type): - try: - self.type = ['buymf', 'sellmf', 'reinvest', 'buystock', 'sellstock'].index(type.lower()) - except ValueError: - self.type = InvestmentTransaction.Unknown + self.type = type.lower() self.tradeDate = None self.settleDate = None + self.memo = '' self.security = '' + self.income_type = '' self.units = decimal.Decimal(0) self.unit_price = decimal.Decimal(0) + self.commission = decimal.Decimal(0) + self.fees = decimal.Decimal(0) + self.total = decimal.Decimal(0) def __repr__(self): - return "<InvestmentTransaction type=" + str(self.type) + ", units=" + str(self.units) + ">" + return "<InvestmentTransaction type=" + str(self.type) + ", \ + units=" + str(self.units) + ">" class Position(object): @@ -321,8 +388,9 @@ class OfxParser(object): ''' cls_.fail_fast = fail_fast - if isinstance(file_handle, type('')): - raise RuntimeError(six.u("parse() takes in a file handle, not a string")) + if not hasattr(file_handle, 'seek'): + raise TypeError(six.u('parse() accepts a seek-able file handle\ + , not %s' % type(file_handle).__name__)) ofx_obj = Ofx() @@ -332,14 +400,30 @@ class OfxParser(object): ofx_obj.accounts = [] ofx_obj.signon = None + skip_headers(ofx_file.fh) ofx = soup_maker(ofx_file.fh) - if len(ofx.contents) == 0: + if ofx.find('ofx') is None: raise OfxParserException('The ofx file is empty!') sonrs_ofx = ofx.find('sonrs') if sonrs_ofx: ofx_obj.signon = cls_.parseSonrs(sonrs_ofx) + stmttrnrs = ofx.find('stmttrnrs') + if stmttrnrs: + stmttrnrs_trnuid = stmttrnrs.find('trnuid') + if stmttrnrs_trnuid: + ofx_obj.trnuid = stmttrnrs_trnuid.contents[0].strip() + + stmttrnrs_status = stmttrnrs.find('status') + if stmttrnrs_status: + ofx_obj.status = {} + ofx_obj.status['code'] = int( + stmttrnrs_status.find('code').contents[0].strip() + ) + ofx_obj.status['severity'] = \ + stmttrnrs_status.find('severity').contents[0].strip() + stmtrs_ofx = ofx.findAll('stmtrs') if stmtrs_ofx: ofx_obj.accounts += cls_.parseStmtrs(stmtrs_ofx, AccountType.Bank) @@ -386,14 +470,23 @@ class OfxParser(object): timeZoneOffset = datetime.timedelta(hours=tz) + res = re.search("^[0-9]*\.([0-9]{0,5})", ofxDateTime) + if res: + msec = datetime.timedelta(seconds=float("0." + res.group(1))) + else: + msec = datetime.timedelta(seconds=0) + try: local_date = datetime.datetime.strptime( ofxDateTime[:14], '%Y%m%d%H%M%S' ) - return local_date - timeZoneOffset + return local_date - timeZoneOffset + msec except: + if ofxDateTime[:8] == "00000000": + return None + return datetime.datetime.strptime( - ofxDateTime[:8], '%Y%m%d') - timeZoneOffset + ofxDateTime[:8], '%Y%m%d') - timeZoneOffset + msec @classmethod def parseAcctinfors(cls_, acctinfors_ofx, ofx): @@ -462,7 +555,12 @@ class OfxParser(object): name_tag = secinfo_ofx.find('secname') ticker_tag = secinfo_ofx.find('ticker') memo_tag = secinfo_ofx.find('memo') - if uniqueid_tag and name_tag and ticker_tag: + if uniqueid_tag and name_tag: + try: + ticker = ticker_tag.contents[0].strip() + except AttributeError: + # ticker can be empty + ticker = None try: memo = memo_tag.contents[0].strip() except AttributeError: @@ -471,7 +569,7 @@ class OfxParser(object): securityList.append( Security(uniqueid_tag.contents[0].strip(), name_tag.contents[0].strip(), - ticker_tag.contents[0].strip(), + ticker, memo)) return securityList @@ -483,10 +581,10 @@ class OfxParser(object): position.security = tag.contents[0].strip() tag = ofx.find('units') if (hasattr(tag, 'contents')): - position.units = decimal.Decimal(tag.contents[0].strip()) + position.units = cls_.toDecimal(tag) tag = ofx.find('unitprice') if (hasattr(tag, 'contents')): - position.unit_price = decimal.Decimal(tag.contents[0].strip()) + position.unit_price = cls_.toDecimal(tag) tag = ofx.find('dtpriceasof') if (hasattr(tag, 'contents')): try: @@ -521,12 +619,27 @@ class OfxParser(object): tag = ofx.find('uniqueid') if (hasattr(tag, 'contents')): transaction.security = tag.contents[0].strip() + tag = ofx.find('incometype') + if (hasattr(tag, 'contents')): + transaction.income_type = tag.contents[0].strip() tag = ofx.find('units') if (hasattr(tag, 'contents')): - transaction.units = decimal.Decimal(tag.contents[0].strip()) + transaction.units = cls_.toDecimal(tag) tag = ofx.find('unitprice') if (hasattr(tag, 'contents')): - transaction.unit_price = decimal.Decimal(tag.contents[0].strip()) + transaction.unit_price = cls_.toDecimal(tag) + tag = ofx.find('commission') + if (hasattr(tag, 'contents')): + transaction.commission = cls_.toDecimal(tag) + tag = ofx.find('fees') + if (hasattr(tag, 'contents')): + transaction.fees = cls_.toDecimal(tag) + tag = ofx.find('total') + if (hasattr(tag, 'contents')): + transaction.total = cls_.toDecimal(tag) + tag = ofx.find('inv401ksource') + if (hasattr(tag, 'contents')): + transaction.inv401ksource = tag.contents[0].strip() return transaction @classmethod @@ -548,7 +661,8 @@ class OfxParser(object): raise except ValueError: e = sys.exc_info()[1] - statement.warnings.append(six.u('Invalid start date: %s') % e) + statement.warnings.append(six.u('Invalid start date:\ + %s') % e) if cls_.fail_fast: raise @@ -561,7 +675,8 @@ class OfxParser(object): statement.warnings.append(six.u('Empty end date.')) except ValueError: e = sys.exc_info()[1] - statement.warnings.append(six.u('Invalid end date: %s') % e) + statement.warnings.append(six.u('Invalid end date: \ + %s') % e) if cls_.fail_fast: raise @@ -576,12 +691,11 @@ class OfxParser(object): if cls_.fail_fast: raise statement.discarded_entries.append( - {six.u('error'): six.u("Error parsing positions: ") + str(e), - six.u('content'): investment_ofx} + {six.u('error'): six.u("Error parsing positions: \ + ") + str(e), six.u('content'): investment_ofx} ) - for transaction_type in ['buymf', 'sellmf', 'reinvest', 'buystock', - 'sellstock', 'buyopt', 'sellopt']: + for transaction_type in InvestmentTransaction.AGGREGATE_TYPES: try: for investment_ofx in invstmtrs_ofx.findAll(transaction_type): statement.transactions.append( @@ -613,14 +727,28 @@ class OfxParser(object): @classmethod def parseSonrs(cls_, sonrs): - code = int(sonrs.find('code').contents[0].strip()) - severity = sonrs.find('severity').contents[0].strip() - try: - message = sonrs.find('message').contents[0].strip() - except: - message = '' + items = [ + 'code', + 'severity', + 'dtserver', + 'language', + 'dtprofup', + 'org', + 'fid', + 'intu.bid', + 'message' + ] + idict = {} + for i in items: + try: + idict[i] = sonrs.find(i).contents[0].strip() + except: + idict[i] = None + idict['code'] = int(idict['code']) + if idict['message'] is None: + idict['message'] = '' - return Signon(code,severity,message) + return Signon(idict) @classmethod def parseStmtrs(cls_, stmtrs_list, accountType): @@ -628,6 +756,9 @@ class OfxParser(object): ret = [] for stmtrs_ofx in stmtrs_list: account = Account() + act_curdef = stmtrs_ofx.find('curdef') + if act_curdef: + account.curdef = act_curdef.contents[0].strip() acctid_tag = stmtrs_ofx.find('acctid') if hasattr(acctid_tag, 'contents'): account.account_id = acctid_tag.contents[0].strip() @@ -648,6 +779,41 @@ class OfxParser(object): return ret @classmethod + def parseBalance(cls_, statement, stmt_ofx, bal_tag_name, bal_attr, + bal_date_attr, bal_type_string): + bal_tag = stmt_ofx.find(bal_tag_name) + if hasattr(bal_tag, "contents"): + balamt_tag = bal_tag.find('balamt') + dtasof_tag = bal_tag.find('dtasof') + if hasattr(balamt_tag, "contents"): + try: + setattr(statement, bal_attr, cls_.toDecimal(balamt_tag)) + except (IndexError, decimal.InvalidOperation): + ex = sys.exc_info()[1] + statement.warnings.append( + six.u("%s balance amount was empty for \ + %s") % (bal_type_string, stmt_ofx)) + if cls_.fail_fast: + raise OfxParserException("Empty %s balance\ + " % bal_type_string) + if hasattr(dtasof_tag, "contents"): + try: + setattr(statement, bal_date_attr, cls_.parseOfxDateTime( + dtasof_tag.contents[0].strip())) + except IndexError: + statement.warnings.append( + six.u("%s balance date was empty for %s\ + ") % (bal_type_string, stmt_ofx)) + if cls_.fail_fast: + raise + except ValueError: + statement.warnings.append( + six.u("%s balance date was not allowed for \ + %s") % (bal_type_string, stmt_ofx)) + if cls_.fail_fast: + raise + + @classmethod def parseStatement(cls_, stmt_ofx): ''' Parse a statement in ofx-land and return a Statement object. @@ -665,7 +831,8 @@ class OfxParser(object): raise except ValueError: statement.warnings.append( - six.u("Statement start date was not allowed for %s") % stmt_ofx) + six.u("Statement start date was not allowed for \ + %s") % stmt_ofx) if cls_.fail_fast: raise @@ -682,13 +849,14 @@ class OfxParser(object): except ValueError: ve = sys.exc_info()[1] msg = six.u("Statement start date was not formatted " - "correctly for %s") + "correctly for %s") statement.warnings.append(msg % stmt_ofx) if cls_.fail_fast: raise except TypeError: statement.warnings.append( - six.u("Statement start date was not allowed for %s") % stmt_ofx) + six.u("Statement start date was not allowed for \ + %s") % stmt_ofx) if cls_.fail_fast: raise @@ -702,33 +870,11 @@ class OfxParser(object): if cls_.fail_fast: raise - ledger_bal_tag = stmt_ofx.find('ledgerbal') - if hasattr(ledger_bal_tag, "contents"): - balamt_tag = ledger_bal_tag.find('balamt') - if hasattr(balamt_tag, "contents"): - try: - statement.balance = decimal.Decimal( - balamt_tag.contents[0].strip()) - except (IndexError, decimal.InvalidOperation): - ex = sys.exc_info()[1] - statement.warnings.append( - six.u("Ledger balance amount was empty for %s") % stmt_ofx) - if cls_.fail_fast: - raise OfxParserException("Empty ledger balance") + cls_.parseBalance(statement, stmt_ofx, 'ledgerbal', + 'balance', 'balance_date', 'ledger') - avail_bal_tag = stmt_ofx.find('availbal') - if hasattr(avail_bal_tag, "contents"): - balamt_tag = avail_bal_tag.find('balamt') - if hasattr(balamt_tag, "contents"): - try: - statement.available_balance = decimal.Decimal( - balamt_tag.contents[0].strip()) - except (IndexError, decimal.InvalidOperation): - ex = sys.exc_info()[1] - msg = six.u("Available balance amount was empty for %s") - statement.warnings.append(msg % stmt_ofx) - if cls_.fail_fast: - raise OfxParserException("Empty available balance") + cls_.parseBalance(statement, stmt_ofx, 'availbal', 'available_balance', + 'available_balance_date', 'ledger') for transaction_ofx in stmt_ofx.findAll('stmttrn'): try: @@ -783,13 +929,17 @@ class OfxParser(object): amt_tag = txn_ofx.find('trnamt') if hasattr(amt_tag, "contents"): try: - transaction.amount = decimal.Decimal( - amt_tag.contents[0].strip()) + transaction.amount = cls_.toDecimal(amt_tag) except IndexError: raise OfxParserException("Invalid Transaction Date") except decimal.InvalidOperation: - raise OfxParserException( - six.u("Invalid Transaction Amount: '%s'") % amt_tag.contents[0]) + # Some banks use a null transaction for including interest + # rate changes on your statement. + if amt_tag.contents[0].strip() in ('null', '-null'): + transaction.amount = 0 + else: + raise OfxParserException( + six.u("Invalid Transaction Amount: '%s'") % amt_tag.contents[0]) except TypeError: raise OfxParserException( six.u("No Transaction Amount (a required field)")) @@ -819,24 +969,29 @@ class OfxParser(object): try: transaction.id = id_tag.contents[0].strip() except IndexError: - raise OfxParserException(six.u("Empty FIT id (a required field)")) + raise OfxParserException(six.u("Empty FIT id (a required \ + field)")) except TypeError: raise OfxParserException(six.u("No FIT id (a required field)")) else: - raise OfxParserException(six.u("Missing FIT id (a required field)")) + raise OfxParserException(six.u("Missing FIT id (a required \ + field)")) sic_tag = txn_ofx.find('sic') if hasattr(sic_tag, 'contents'): try: transaction.sic = sic_tag.contents[0].strip() except IndexError: - raise OfxParserException(six.u("Empty transaction Standard Industry Code (SIC)")) + raise OfxParserException(six.u("Empty transaction Standard \ + Industry Code (SIC)")) if transaction.sic is not None and transaction.sic in mcc.codes: try: - transaction.mcc = mcc.codes.get(transaction.sic, '').get('combined description') + transaction.mcc = mcc.codes.get(transaction.sic, '').get('combined \ + description') except IndexError: - raise OfxParserException(six.u("Empty transaction Merchant Category Code (MCC)")) + raise OfxParserException(six.u("Empty transaction Merchant Category \ + Code (MCC)")) except AttributeError: if cls._fail_fast: raise @@ -846,6 +1001,14 @@ class OfxParser(object): try: transaction.checknum = checknum_tag.contents[0].strip() except IndexError: - raise OfxParserException(six.u("Empty Check (or other reference) number")) + raise OfxParserException(six.u("Empty Check (or other reference) \ + number")) return transaction + + @classmethod + def toDecimal(cls_, tag): + d = tag.contents[0].strip() + if '.' not in d and ',' in d: + d = d.replace(',', '.') + return decimal.Decimal(d) diff --git a/ofxparse/ofxprinter.py b/ofxparse/ofxprinter.py new file mode 100644 index 0000000..4c0a13d --- /dev/null +++ b/ofxparse/ofxprinter.py @@ -0,0 +1,194 @@ +class OfxPrinter(): + ofx = None + out_filename = None + out_handle = None + term = "\r\n" + + def __init__(self, ofx, filename, term="\r\n"): + self.ofx = ofx + self.out_filename = filename + self.term = term + + def writeLine(self, data, tabs=0, term=None): + if term is None: + term = self.term + + tabbing = (tabs * "\t") if (tabs > 0) else '' + + return self.out_handle.write( + "{0}{1}{2}".format( + tabbing, + data, + term + ) + ) + + def writeHeaders(self): + for k, v in self.ofx.headers.iteritems(): + if v is None: + self.writeLine("{0}:NONE".format(k)) + else: + self.writeLine("{0}:{1}".format(k, v)) + self.writeLine("") + + def writeSignOn(self, tabs=0): + # signon already has newlines and tabs in it + # TODO: reimplement signon printing with tabs + self.writeLine(self.ofx.signon.__str__(), term="") + + def printDate(self, dt, msec_digs=3): + strdt = dt.strftime('%Y%m%d%H%M%S') + strdt_msec = dt.strftime('%f') + if len(strdt_msec) < msec_digs: + strdt_msec += ('0' * (msec_digs - len(strdt_msec))) + elif len(strdt_msec) > msec_digs: + strdt_msec = strdt_msec[:msec_digs] + return strdt + '.' + strdt_msec + + def writeTrn(self, trn, tabs=5): + self.writeLine("<STMTTRN>", tabs=tabs) + tabs += 1 + + self.writeLine("<TRNTYPE>{}".format(trn.type.upper()), tabs=tabs) + self.writeLine("<DTPOSTED>{}".format( + self.printDate(trn.date) + ), tabs=tabs) + self.writeLine("<TRNAMT>{0:.2f}".format(float(trn.amount)), tabs=tabs) + + self.writeLine("<FITID>{}".format(trn.id), tabs=tabs) + + if len(str(trn.checknum)) > 0: + self.writeLine("<CHECKNUM>{}".format( + trn.checknum + ), tabs=tabs) + + self.writeLine("<NAME>{}".format(trn.payee), tabs=tabs) + + if len(trn.memo.strip()) > 0: + self.writeLine("<MEMO>{}".format(trn.memo), tabs=tabs) + + tabs -= 1 + self.writeLine("</STMTTRN>", tabs=tabs) + + def writeLedgerBal(self, statement, tabs=4): + bal = getattr(statement, 'balance') + baldt = getattr(statement, 'balance_date') + + if bal and baldt: + self.writeLine("<LEDGERBAL>", tabs=tabs) + self.writeLine("<BALAMT>{0:.2f}".format(float(bal)), tabs=tabs+1) + self.writeLine("<DTASOF>{0}".format( + self.printDate(baldt) + ), tabs=tabs+1) + self.writeLine("</LEDGERBAL>", tabs=tabs) + + def writeAvailBal(self, statement, tabs=4): + bal = getattr(statement, 'available_balance') + baldt = getattr(statement, 'available_balance_date') + + if bal and baldt: + self.writeLine("<AVAILBAL>", tabs=tabs) + self.writeLine("<BALAMT>{0:.2f}".format(float(bal)), tabs=tabs+1) + self.writeLine("<DTASOF>{0}".format( + self.printDate(baldt) + ), tabs=tabs+1) + self.writeLine("</AVAILBAL>", tabs=tabs) + + def writeStmTrs(self, tabs=3): + for acct in self.ofx.accounts: + self.writeLine("<STMTRS>", tabs=tabs) + tabs += 1 + + if acct.curdef: + self.writeLine("<CURDEF>{0}".format( + acct.curdef + ), tabs=tabs) + + if acct.routing_number or acct.account_id or acct.account_type: + self.writeLine("<BANKACCTFROM>", tabs=tabs) + if acct.routing_number: + self.writeLine("<BANKID>{0}".format( + acct.routing_number + ), tabs=tabs+1) + if acct.account_id: + self.writeLine("<ACCTID>{0}".format( + acct.account_id + ), tabs=tabs+1) + if acct.account_type: + self.writeLine("<ACCTTYPE>{0}".format( + acct.account_type + ), tabs=tabs+1) + self.writeLine("</BANKACCTFROM>", tabs=tabs) + + self.writeLine("<BANKTRANLIST>", tabs=tabs) + tabs += 1 + self.writeLine("<DTSTART>{0}".format( + self.printDate(acct.statement.start_date) + ), tabs=tabs) + self.writeLine("<DTEND>{0}".format( + self.printDate(acct.statement.end_date) + ), tabs=tabs) + + for trn in acct.statement.transactions: + self.writeTrn(trn, tabs=tabs) + + tabs -= 1 + + self.writeLine("</BANKTRANLIST>", tabs=tabs) + + self.writeLedgerBal(acct.statement, tabs=tabs) + self.writeAvailBal(acct.statement, tabs=tabs) + + tabs -= 1 + + self.writeLine("</STMTRS>", tabs=tabs) + + def writeBankMsgsRsv1(self, tabs=1): + self.writeLine("<BANKMSGSRSV1>", tabs=tabs) + tabs += 1 + self.writeLine("<STMTTRNRS>", tabs=tabs) + tabs += 1 + if self.ofx.trnuid is not None: + self.writeLine("<TRNUID>{0}".format( + self.ofx.trnuid + ), tabs=tabs) + if self.ofx.status: + self.writeLine("<STATUS>", tabs=tabs) + self.writeLine("<CODE>{0}".format( + self.ofx.status['code'] + ), tabs=tabs+1) + self.writeLine("<SEVERITY>{0}".format( + self.ofx.status['severity'] + ), tabs=tabs+1) + self.writeLine("</STATUS>", tabs=tabs) + self.writeStmTrs(tabs=tabs) + tabs -= 1 + self.writeLine("</STMTTRNRS>", tabs=tabs) + tabs -= 1 + self.writeLine("</BANKMSGSRSV1>", tabs=tabs) + + def writeOfx(self, tabs=0): + self.writeLine("<OFX>", tabs=tabs) + tabs += 1 + self.writeSignOn(tabs=tabs) + self.writeBankMsgsRsv1(tabs=tabs) + tabs -= 1 + # No newline at end of file + self.writeLine("</OFX>", tabs=tabs, term="") + + def write(self, filename=None, tabs=0): + if self.out_handle: + raise Exception("Already writing file") + + if filename is None: + filename = self.out_filename + + self.out_handle = open(filename, 'wb') + + self.writeHeaders() + + self.writeOfx(tabs=tabs) + + self.out_handle.flush() + self.out_handle.close() + self.out_handle = None diff --git a/ofxparse/ofxutil.py b/ofxparse/ofxutil.py index 17c2f36..0002ee1 100644 --- a/ofxparse/ofxutil.py +++ b/ofxparse/ofxutil.py @@ -12,9 +12,11 @@ else: import six + class InvalidOFXStructureException(Exception): pass + class OfxData(object): def __init__(self, tag): self.nodes = odict.OrderedDict() @@ -38,7 +40,8 @@ class OfxData(object): del self.nodes[name] def __setattr__(self, name, value): - if name in self.__dict__ or name in ['nodes', 'tag', 'data', 'headers', 'xml']: + if name in self.__dict__ or name in ['nodes', 'tag', 'data', '\ + headers', 'xml']: self.__dict__[name] = value else: self.del_tag(name) @@ -100,12 +103,14 @@ class OfxData(object): return len(self.nodes) def __str__(self): - return os.linesep.join("\t" * line[1] + line[0] for line in self.format()) + return os.linesep.join("\t" * line[1] + line[0] for line \ + in self.format()) def format(self): if self.data or not self.nodes: if self.tag.upper() == "OFX": - return [["<%s>%s</%s>" % (self.tag, self.data if self.data else "", self.tag), 0]] + return [["<%s>%s</%s>" % (self.tag, self.data \ + if self.data else "", self.tag), 0]] return [["<%s>%s" % (self.tag, self.data), 0]] else: ret = [["<%s>" % self.tag, -1]] @@ -129,10 +134,12 @@ class OfxUtil(OfxData): self.headers = odict.OrderedDict() self.xml = "" if ofx_data: - if isinstance(ofx_data, six.string_types) and not ofx_data.lower().endswith('.ofx'): + if isinstance(ofx_data, six.string_types) and not \ + ofx_data.lower().endswith('.ofx'): self.parse(ofx_data) else: - self.parse(open(ofx_data).read() if isinstance(ofx_data, six.string_types) else ofx_data.read()) + self.parse(open(ofx_data).read() if isinstance(\ + ofx_data, six.string_types) else ofx_data.read()) def parse(self, ofx): try: @@ -168,24 +175,27 @@ class OfxUtil(OfxData): for i, tag in enumerate(tags): gt = tag.index(">") if tag[1] != "/": - #Is an opening tag + # Is an opening tag if not can_open: - tags[i - 1] = tags[i - 1] + "</" + heirarchy.pop() + ">" + tags[i - 1] = tags[i - 1] + "</" + \ + heirarchy.pop() + ">" can_open = True tag_name = tag[1:gt].split()[0] heirarchy.append(tag_name) if len(tag) > gt + 1: can_open = False else: - #Is a closing tag + # Is a closing tag tag_name = tag[2:gt].split()[0] if tag_name not in heirarchy: - #Close tag with no matching open, so delete it + # Close tag with no matching open, so delete it tags[i] = tag[gt + 1:] else: - #Close tag with matching open, but other open tags that need to be closed first + # Close tag with matching open, but other open + # tags that need to be closed first while(tag_name != heirarchy[-1]): - tags[i - 1] = tags[i - 1] + "</" + heirarchy.pop() + ">" + tags[i - 1] = tags[i - 1] + "</" + \ + heirarchy.pop() + ">" can_open = True heirarchy.pop() @@ -209,7 +219,8 @@ class OfxUtil(OfxData): f.write(str(self)) def __str__(self): - ret = os.linesep.join(":".join(line) for line in six.iteritems(self.headers)) + os.linesep * 2 + ret = os.linesep.join(":".join(line) for line in \ + six.iteritems(self.headers)) + os.linesep * 2 ret += super(OfxUtil, self).__str__() return ret @@ -219,22 +230,20 @@ if __name__ == "__main__": ofx = OfxUtil(fixtures + 'checking.ofx') # ofx = OfxUtil(fixtures + 'fidelity.ofx') - - #Manipulate OFX file via XML library + # Manipulate OFX file via XML library # for transaction in ofx.xml.iter('STMTTRN'): # transaction.find('NAME').text = transaction.find('MEMO').text # transaction.remove(transaction.find('MEMO')) # ofx.reload_xml() - - #Manipulate OFX file via object tree built from XML + # Manipulate OFX file via object tree built from XML # for transaction in ofx.bankmsgsrsv1.stmttrnrs.stmtrs.banktranlist.stmttrn: # transaction.name = transaction.memo # del transaction.memo # transaction.notes = "Acknowledged" - - #Modified sytnax for object tree data manipulation - #I'm using the __getitem__ method like the xml.iter method from ElementTree, as a recursive search + # Modified sytnax for object tree data manipulation + # I'm using the __getitem__ method like the xml.iter method from + # ElementTree, as a recursive search for transaction in ofx['stmttrn']: transaction.name = transaction.memo del transaction.memo @@ -248,7 +257,7 @@ if __name__ == "__main__": # print(ofx) - #Write OFX data to output file + # Write OFX data to output file # ofx.write('out.ofx') # for file_name in os.listdir(fixtures): @@ -1,3 +1,6 @@ + +import codecs +import os import re import sys @@ -10,30 +13,37 @@ VERSION = re.search(r"__version__ = '(.*?)'", open("ofxparse/__init__.py").read()).group(1) # Use BeautifulSoup 3 on Python 2.5 and earlier and BeautifulSoup 4 otherwise -if sys.version_info < (2,6): +if sys.version_info < (2, 6): REQUIRES = [ "beautifulSoup>=3.0", ] else: REQUIRES = [ - "beautifulsoup4" + "beautifulsoup4", + "lxml", ] -if sys.version_info < (2,7): +if sys.version_info < (2, 7): REQUIRES.extend([ "ordereddict>=1.1", ]) REQUIRES.extend([ 'six', + 'lxml' ]) +README = os.path.join(os.path.dirname(__file__), 'README.rst') + +with codecs.open(README, encoding='utf8') as f: + LONG_DESCRIPTION = f.read() + setup_params = dict( name='ofxparse', version=VERSION, description=("Tools for working with the OFX (Open Financial Exchange)" " file format"), - long_description=open("./README", "r").read(), + long_description=LONG_DESCRIPTION, # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers classifiers=[ "Development Status :: 4 - Beta", diff --git a/tests/fixtures/investment_401k.ofx b/tests/fixtures/investment_401k.ofx new file mode 100644 index 0000000..1fc0898 --- /dev/null +++ b/tests/fixtures/investment_401k.ofx @@ -0,0 +1,180 @@ +OFXHEADER:100 +DATA:OFXSGML +VERSION:102 +SECURITY:NONE +ENCODING:USASCII +CHARSET:1252 +COMPRESSION:NONE +OLDFILEUID:NONE +NEWFILEUID:NONE + +<OFX> + <SIGNONMSGSRSV1> + <SONRS> + <STATUS> + <CODE>0</CODE> + <SEVERITY>INFO</SEVERITY> + <MESSAGE>SUCCESS</MESSAGE> + </STATUS> + <DTSERVER>20150909084609.717[-6:MDT]</DTSERVER> + <LANGUAGE>ENG</LANGUAGE> + <FI> + <ORG>EXAMPLE</ORG> + <FID>1234</FID> + </FI> + <INTU.BID>1234</INTU.BID> + </SONRS> + </SIGNONMSGSRSV1> + <INVSTMTMSGSRSV1> + <INVSTMTTRNRS> + <TRNUID>0</TRNUID> + <STATUS> + <CODE>0</CODE> + <SEVERITY>INFO</SEVERITY> + <MESSAGE>SUCCESS</MESSAGE> + </STATUS> + <INVSTMTRS> + <DTASOF>20140630000000.000[-6:MDT]</DTASOF> + <CURDEF>USD</CURDEF> + <INVACCTFROM> + <BROKERID>example.org</BROKERID> + <ACCTID>12345678.123456-01</ACCTID> + </INVACCTFROM> + <INVTRANLIST> + <DTSTART>20140401000000.000[-6:MDT]</DTSTART> + <DTEND>20140630000000.000[-6:MDT]</DTEND> + <BUYMF> + <INVBUY> + <INVTRAN> + <FITID>1</FITID> + <DTTRADE>20140617000000.000[-6:MDT]</DTTRADE> + </INVTRAN> + <SECID> + <UNIQUEID>FOO</UNIQUEID> + <UNIQUEIDTYPE>PRIVATE</UNIQUEIDTYPE> + </SECID> + <UNITS>8.846699</UNITS> + <UNITPRICE>22.2908</UNITPRICE> + <TOTAL>-197.2</TOTAL> + <SUBACCTSEC>OTHER</SUBACCTSEC> + <SUBACCTFUND>OTHER</SUBACCTFUND> + </INVBUY> + <BUYTYPE>BUY</BUYTYPE> + </BUYMF> + <TRANSFER> + <INVTRAN> + <FITID>2</FITID> + <DTTRADE>20140630000000.000[-6:MDT]</DTTRADE> + </INVTRAN> + <SECID> + <UNIQUEID>BAR</UNIQUEID> + <UNIQUEIDTYPE>PRIVATE</UNIQUEIDTYPE> + </SECID> + <SUBACCTSEC>OTHER</SUBACCTSEC> + <UNITS>6.800992</UNITS> + <TFERACTION>IN</TFERACTION> + <POSTYPE>LONG</POSTYPE> + <UNITPRICE>29.214856</UNITPRICE> + </TRANSFER> + <TRANSFER> + <INVTRAN> + <FITID>3</FITID> + <DTTRADE>20140630000000.000[-6:MDT]</DTTRADE> + </INVTRAN> + <SECID> + <UNIQUEID>BAZ</UNIQUEID> + <UNIQUEIDTYPE>PRIVATE</UNIQUEIDTYPE> + </SECID> + <SUBACCTSEC>OTHER</SUBACCTSEC> + <UNITS>-9.060702</UNITS> + <TFERACTION>OUT</TFERACTION> + <POSTYPE>LONG</POSTYPE> + <UNITPRICE>21.928764</UNITPRICE> + </TRANSFER> + </INVTRANLIST> + <INVPOSLIST> + <POSMF> + <INVPOS> + <SECID> + <UNIQUEID>FOO</UNIQUEID> + <UNIQUEIDTYPE>PRIVATE</UNIQUEIDTYPE> + </SECID> + <HELDINACCT>CASH</HELDINACCT> + <POSTYPE>LONG</POSTYPE> + <UNITS>17.604312</UNITS> + <UNITPRICE>22.517211</UNITPRICE> + <MKTVAL>396.4</MKTVAL> + <DTPRICEASOF>20140630000000.000[-6:MDT]</DTPRICEASOF> + </INVPOS> + </POSMF> + <POSMF> + <INVPOS> + <SECID> + <UNIQUEID>BAR</UNIQUEID> + <UNIQUEIDTYPE>PRIVATE</UNIQUEIDTYPE> + </SECID> + <HELDINACCT>CASH</HELDINACCT> + <POSTYPE>LONG</POSTYPE> + <UNITS>13.550983</UNITS> + <UNITPRICE>29.214855</UNITPRICE> + <MKTVAL>395.89</MKTVAL> + <DTPRICEASOF>20140630000000.000[-6:MDT]</DTPRICEASOF> + </INVPOS> + </POSMF> + <POSMF> + <INVPOS> + <SECID> + <UNIQUEID>BAZ</UNIQUEID> + <UNIQUEIDTYPE>PRIVATE</UNIQUEIDTYPE> + </SECID> + <HELDINACCT>CASH</HELDINACCT> + <POSTYPE>LONG</POSTYPE> + <UNITS>0.0</UNITS> + <UNITPRICE>0.0</UNITPRICE> + <MKTVAL>0.0</MKTVAL> + <DTPRICEASOF>20140630000000.000[-6:MDT]</DTPRICEASOF> + </INVPOS> + </POSMF> + </INVPOSLIST> + <INV401KBAL> + <TOTAL>1000.00</TOTAL> + </INV401KBAL> + </INVSTMTRS> + </INVSTMTTRNRS> + </INVSTMTMSGSRSV1> + <SECLISTMSGSRSV1> + <SECLIST> + <MFINFO> + <SECINFO> + <SECID> + <UNIQUEID>BAR</UNIQUEID> + <UNIQUEIDTYPE>PRIVATE</UNIQUEIDTYPE> + </SECID> + <SECNAME>BAR Index Fund</SECNAME> + <TICKER>BAR</TICKER> + </SECINFO> + </MFINFO> + <MFINFO> + <SECINFO> + <SECID> + <UNIQUEID>FOO</UNIQUEID> + <UNIQUEIDTYPE>PRIVATE</UNIQUEIDTYPE> + </SECID> + <SECNAME>Foo Index Fund</SECNAME> + <TICKER>FOO</TICKER> + </SECINFO> + </MFINFO> + <MFINFO> + <SECINFO> + <SECID> + <UNIQUEID>BAZ</UNIQUEID> + <UNIQUEIDTYPE>PRIVATE</UNIQUEIDTYPE> + </SECID> + <SECNAME>Baz Fund</SECNAME> + <TICKER>BAZ</TICKER> + </SECINFO> + </MFINFO> + </SECLIST> + </SECLISTMSGSRSV1> +</OFX> + diff --git a/tests/fixtures/suncorp.ofx b/tests/fixtures/suncorp.ofx new file mode 100644 index 0000000..4a0558d --- /dev/null +++ b/tests/fixtures/suncorp.ofx @@ -0,0 +1,56 @@ +<?xml version="1.0" encoding="us-ascii"?>
+<?OFX OFXHEADER="200" VERSION="200" SECURITY="NONE" OLDFILEUID="NONE" NEWFILEUID="NONE"?>
+<OFX>
+ <SIGNONMSGSRSV1>
+ <SONRS>
+ <STATUS>
+ <CODE>0</CODE>
+ <SEVERITY>INFO</SEVERITY>
+ </STATUS>
+ <DTSERVER>20131215</DTSERVER>
+ <LANGUAGE>ENG</LANGUAGE>
+ <FI>
+ <ORG>SUNCORP</ORG>
+ <FID>484-799</FID>
+ </FI>
+ </SONRS>
+ </SIGNONMSGSRSV1>
+ <BANKMSGSRSV1>
+ <STMTTRNRS>
+ <TRNUID>1</TRNUID>
+ <STATUS>
+ <CODE>0</CODE>
+ <SEVERITY>INFO</SEVERITY>
+ </STATUS>
+ <STMTRS>
+ <CURDEF>AUD</CURDEF>
+ <BANKACCTFROM>
+ <BANKID>SUNCORP</BANKID>
+ <ACCTID>123456789</ACCTID>
+ <ACCTTYPE>CHECKING</ACCTTYPE>
+ </BANKACCTFROM>
+ <BANKTRANLIST>
+ <DTSTART>20130618</DTSTART>
+ <DTEND>20131215</DTEND>
+ <STMTTRN>
+ <TRNTYPE>DEBIT</TRNTYPE>
+ <DTPOSTED>20131215</DTPOSTED>
+ <TRNAMT>-16.85</TRNAMT>
+ <FITID>1</FITID>
+ <CHECKNUM>0</CHECKNUM>
+ <NAME><![CDATA[EFTPOS WDL HANDYWAY ALDI STORE ]]></NAME>
+ <MEMO><![CDATA[EFTPOS WDL HANDYWAY ALDI STORE GEELONG WEST VICAU]]></MEMO>
+ </STMTTRN>
+ </BANKTRANLIST>
+ <LEDGERBAL>
+ <BALAMT>1234.12</BALAMT>
+ <DTASOF>20131215</DTASOF>
+ </LEDGERBAL>
+ <AVAILBAL>
+ <BALAMT>1234.12</BALAMT>
+ <DTASOF>20131215</DTASOF>
+ </AVAILBAL>
+ </STMTRS>
+ </STMTTRNRS>
+ </BANKMSGSRSV1>
+</OFX>
\ No newline at end of file diff --git a/tests/fixtures/vanguard401k.ofx b/tests/fixtures/vanguard401k.ofx new file mode 100644 index 0000000..0443ca1 --- /dev/null +++ b/tests/fixtures/vanguard401k.ofx @@ -0,0 +1,11 @@ +OFXHEADER:100 +DATA:OFXSGML +VERSION:102 +SECURITY:NONE +ENCODING:USASCII +CHARSET:1252 +COMPRESSION:NONE +OLDFILEUID:NONE +NEWFILEUID:NONE + +<OFX><SIGNONMSGSRSV1><SONRS><STATUS><CODE>0<SEVERITY>INFO<MESSAGE>Successful Sign On</STATUS><DTSERVER>20141018150740[-5:EST]<LANGUAGE>ENG<DTPROFUP>20140605083000<FI><ORG>Vanguard<FID>84022</FI><SESSCOOKIE>foo<INTU.BID>84022<INTU.USERID>USER345</SONRS></SIGNONMSGSRSV1><INVSTMTMSGSRSV1><INVSTMTTRNRS><TRNUID>0<STATUS><CODE>0<SEVERITY>INFO</STATUS><INVSTMTRS><DTASOF>20141017160000.000[-5:EST]<CURDEF>USD<INVACCTFROM><BROKERID>vanguard.com<ACCTID>0123456</INVACCTFROM><INVTRANLIST><DTSTART>20140916160000.000[-5:EST]<DTEND>20141018150740.000[-5:EST]<BUYMF><INVBUY><INVTRAN><FITID>1234567890123456790AAA<DTTRADE>20140926160000.000[-5:EST]<DTSETTLE>20140926160000.000[-5:EST]<MEMO>Price as of date based on closing price</INVTRAN><SECID><UNIQUEID>92202V351<UNIQUEIDTYPE>CUSIP</SECID><UNITS>14.61137<UNITPRICE>46.06<TOTAL>-673.0<SUBACCTSEC>CASH<SUBACCTFUND>OTHER<INV401KSOURCE>PRETAX</INVBUY><BUYTYPE>BUY</BUYMF><BUYMF><INVBUY><INVTRAN><FITID>1234567890123456791AAA<DTTRADE>20140926160000.000[-5:EST]<DTSETTLE>20140926160000.000[-5:EST]<MEMO>Price as of date based on closing price</INVTRAN><SECID><UNIQUEID>92202V351<UNIQUEIDTYPE>CUSIP</SECID><UNITS>7.30568<UNITPRICE>46.06<TOTAL>-336.5<SUBACCTSEC>CASH<SUBACCTFUND>OTHER<INV401KSOURCE>MATCH</INVBUY><BUYTYPE>BUY</BUYMF><BUYMF><INVBUY><INVTRAN><FITID>1234567890123456793AAA<DTTRADE>20141010160000.000[-5:EST]<DTSETTLE>20141010160000.000[-5:EST]<MEMO>Price as of date based on closing price</INVTRAN><SECID><UNIQUEID>92202V351<UNIQUEIDTYPE>CUSIP</SECID><UNITS>15.25039<UNITPRICE>44.13<TOTAL>-673.0<SUBACCTSEC>CASH<SUBACCTFUND>OTHER<INV401KSOURCE>PRETAX</INVBUY><BUYTYPE>BUY</BUYMF><BUYMF><INVBUY><INVTRAN><FITID>1234567890123456794AAA<DTTRADE>20141010160000.000[-5:EST]<DTSETTLE>20141010160000.000[-5:EST]<MEMO>Price as of date based on closing price</INVTRAN><SECID><UNIQUEID>92202V351<UNIQUEIDTYPE>CUSIP</SECID><UNITS>7.62519<UNITPRICE>44.13<TOTAL>-336.5<SUBACCTSEC>CASH<SUBACCTFUND>OTHER<INV401KSOURCE>MATCH</INVBUY><BUYTYPE>BUY</BUYMF><TRANSFER><INVTRAN><FITID>1234567890123456795AAA<DTTRADE>20130905160000.000[-5:EST]<DTSETTLE>20130906160000.000[-5:EST]<MEMO>Investment Expense</INVTRAN><SECID><UNIQUEID>92202V351<UNIQUEIDTYPE>CUSIP</SECID><SUBACCTSEC>CASH<UNITS>-0.04241<TFERACTION>OUT<POSTYPE>LONG<UNITPRICE>39.37<INV401KSOURCE>MATCH</TRANSFER></INVTRANLIST><INVPOSLIST><POSMF><INVPOS><SECID><UNIQUEID>92202V351<UNIQUEIDTYPE>CUSIP</SECID><HELDINACCT>OTHER<POSTYPE>LONG<UNITS>117.506<UNITPRICE>44.01<MKTVAL>5171.44<DTPRICEASOF>20141017160000.000[-5:EST]<MEMO>Price as of date based on closing price<INV401KSOURCE>OTHERNONVEST</INVPOS><REINVDIV>Y<REINVCG>Y</POSMF></INVPOSLIST><INV401K><EMPLOYERNAME>GOOGLE INC. 401(K) SAVINGS PLAN<CURRENTVESTPCT>100.0</INV401K><INV401KBAL><CASHBAL>0.0<PRETAX>0.0<AFTERTAX>0.0<MATCH>0.0<PROFITSHARING>0.0<ROLLOVER>0.0<OTHERVEST>0.0<OTHERNONVEST>0.0<TOTAL>0.0</INV401KBAL></INVSTMTRS></INVSTMTTRNRS></INVSTMTMSGSRSV1><SECLISTMSGSRSV1><SECLIST><MFINFO><SECINFO><SECID><UNIQUEID>92202V351<UNIQUEIDTYPE>CUSIP</SECID><SECNAME>Target Retirement 2050 Trust Plus<FIID>1659<UNITPRICE>44.01<DTASOF>20141017160000.000[-5:EST]<MEMO>Price as of date based on closing price</SECINFO></MFINFO></SECLIST></SECLISTMSGSRSV1></OFX> diff --git a/tests/test_parse.py b/tests/test_parse.py index 35260ce..476b28e 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -1,17 +1,18 @@ from __future__ import absolute_import -from ofxparse.ofxparse import soup_maker +import os from datetime import datetime, timedelta from decimal import Decimal from unittest import TestCase import sys -sys.path.append('..') +sys.path.insert(0, os.path.abspath('..')) import six from .support import open_file from ofxparse import OfxParser, AccountType, Account, Statement, Transaction -from ofxparse.ofxparse import OfxFile, OfxPreprocessedFile, OfxParserException +from ofxparse.ofxparse import OfxFile, OfxPreprocessedFile, OfxParserException, soup_maker + class TestOfxPreprocessedFile(TestCase): @@ -41,8 +42,8 @@ NEWFILEUID:NONE <OFX><DTASOF><![CDATA[></tricky]]><LEAVE ALONE></DTASOF><VAL.UE>a</VAL.UE><VAL_UE>b</VAL_UE><TE_ST></TE_ST><TE.ST></TE.ST><INVBAL><BALLIST><BAL><NAME>Net</NAME><DTASOF>2222</DTASOF></BAL><BAL><NAME>Gross</NAME><DTASOF>3333</DTASOF></BAL></BALLIST></INVBAL></OFX> """ ofx_file = OfxPreprocessedFile(fh) - data = ofx_file.fh.read() - self.assertEqual(data,expect) + data = ofx_file.fh.read() + self.assertEqual(data, expect) def testHeaders(self): expect = {"OFXHEADER": six.u("100"), @@ -124,7 +125,6 @@ NEWFILEUID:NONE self.assertEquals(len(ofx_file.headers.keys()), 2) - class TestOfxFile(TestCase): def testHeaders(self): expect = {"OFXHEADER": six.u("100"), @@ -219,7 +219,7 @@ class TestParse(TestCase): def testThatParseFailsIfAPathIsPassedIn(self): # A file handle should be passed in, not the path. - self.assertRaises(RuntimeError, OfxParser.parse, '/foo/bar') + self.assertRaises(TypeError, OfxParser.parse, '/foo/bar') def testThatParseReturnsAResultWithABankAccount(self): ofx = OfxParser.parse(open_file('bank_medium.ofx')) @@ -232,11 +232,15 @@ class TestParse(TestCase): self.assertEquals('00', ofx.account.branch_id) self.assertEquals('CHECKING', ofx.account.account_type) self.assertEquals(Decimal('382.34'), ofx.account.statement.balance) + self.assertEquals(datetime(2009, 5, 23, 12, 20, 17), + ofx.account.statement.balance_date) # Todo: support values in decimal or int form. # self.assertEquals('15', # ofx.bank_account.statement.balance_in_pennies) self.assertEquals( Decimal('682.34'), ofx.account.statement.available_balance) + self.assertEquals(datetime(2009, 5, 23, 12, 20, 17), + ofx.account.statement.available_balance_date) self.assertEquals( datetime(2009, 4, 1), ofx.account.statement.start_date) self.assertEquals( @@ -271,10 +275,13 @@ class TestStringToDate(TestCase): self.assertRaises(ValueError, OfxParser.parseOfxDateTime, bad_string) bad_but_close_string = '881103' - self.assertRaises(ValueError, OfxParser.parseOfxDateTime, bad_string) + self.assertRaises(ValueError, OfxParser.parseOfxDateTime, bad_but_close_string) no_month_string = '19881301' - self.assertRaises(ValueError, OfxParser.parseOfxDateTime, bad_string) + self.assertRaises(ValueError, OfxParser.parseOfxDateTime, no_month_string) + + def test_returns_none(self): + self.assertIsNone(OfxParser.parseOfxDateTime('00000000')) def test_parses_correct_time(self): '''Test whether it can parse correct time for some valid time fields''' @@ -387,8 +394,58 @@ class TestParseStatement(TestCase): datetime(2009, 5, 23, 12, 20, 17), statement.end_date) self.assertEquals(1, len(statement.transactions)) self.assertEquals(Decimal('382.34'), statement.balance) + self.assertEquals(datetime(2009, 5, 23, 12, 20, 17), statement.balance_date) self.assertEquals(Decimal('682.34'), statement.available_balance) + self.assertEquals(datetime(2009, 5, 23, 12, 20, 17), statement.available_balance_date) + def testThatParseStatementWithBlankDatesReturnsAStatement(self): + input = ''' +<STMTTRNRS> + <TRNUID>20090523122017 + <STATUS> + <CODE>0 + <SEVERITY>INFO + <MESSAGE>OK + </STATUS> + <STMTRS> + <CURDEF>CAD + <BANKACCTFROM> + <BANKID>160000100 + <ACCTID>12300 000012345678 + <ACCTTYPE>CHECKING + </BANKACCTFROM> + <BANKTRANLIST> + <DTSTART>00000000 + <DTEND>00000000 + <STMTTRN> + <TRNTYPE>POS + <DTPOSTED>20090401122017.000[-5:EST] + <TRNAMT>-6.60 + <FITID>0000123456782009040100001 + <NAME>MCDONALD'S #112 + <MEMO>POS MERCHANDISE;MCDONALD'S #112 + </STMTTRN> + </BANKTRANLIST> + <LEDGERBAL> + <BALAMT>382.34 + <DTASOF>20090523122017 + </LEDGERBAL> + <AVAILBAL> + <BALAMT>682.34 + <DTASOF>20090523122017 + </AVAILBAL> + </STMTRS> +</STMTTRNRS> + ''' + txn = soup_maker(input) + statement = OfxParser.parseStatement(txn.find('stmttrnrs')) + self.assertEquals(None, statement.start_date) + self.assertEquals(None, statement.end_date) + self.assertEquals(1, len(statement.transactions)) + self.assertEquals(Decimal('382.34'), statement.balance) + self.assertEquals(datetime(2009, 5, 23, 12, 20, 17), statement.balance_date) + self.assertEquals(Decimal('682.34'), statement.available_balance) + self.assertEquals(datetime(2009, 5, 23, 12, 20, 17), statement.available_balance_date) class TestStatement(TestCase): def testThatANewStatementIsValid(self): @@ -420,7 +477,6 @@ class TestParseTransaction(TestCase): self.assertEquals("MCDONALD'S #112", transaction.payee) self.assertEquals("POS MERCHANDISE;MCDONALD'S #112", transaction.memo) - def testThatParseTransactionWithFieldCheckNum(self): input = ''' <STMTTRN> @@ -436,6 +492,61 @@ class TestParseTransaction(TestCase): transaction = OfxParser.parseTransaction(txn.find('stmttrn')) self.assertEquals('700', transaction.checknum) + def testThatParseTransactionWithCommaAsDecimalPoint(self): + input = ''' +<STMTTRN> + <TRNTYPE>POS + <DTPOSTED>20090401122017.000[-5:EST] + <TRNAMT>-1006,60 + <FITID>0000123456782009040100001 + <NAME>MCDONALD'S #112 + <MEMO>POS MERCHANDISE;MCDONALD'S #112 +</STMTTRN> +''' + txn = soup_maker(input) + transaction = OfxParser.parseTransaction(txn.find('stmttrn')) + self.assertEquals(Decimal('-1006.60'), transaction.amount) + + def testThatParseTransactionWithCommaAsDecimalPointAndDotAsSeparator(self): + input = ''' +<STMTTRN> + <TRNTYPE>POS + <DTPOSTED>20090401122017.000[-5:EST] + <TRNAMT>-1.006,60 + <FITID>0000123456782009040100001 + <NAME>MCDONALD'S #112 + <MEMO>POS MERCHANDISE;MCDONALD'S #112 +</STMTTRN> +''' + txn = soup_maker(input) + with self.assertRaises(OfxParserException): + transaction = OfxParser.parseTransaction(txn.find('stmttrn')) + + def testThatParseTransactionWithNullAmountIgnored(self): + """A null transaction value is converted to 0. + + Some banks use a null transaction to include interest + rate changes on statements. + """ + input_template = ''' +<STMTTRN> + <TRNTYPE>DEP + <DTPOSTED>20130306 + <TRNAMT>{amount} + <FITID>2013030601009100 + <CHECKNUM>700 + <MEMO>DEPOSITO ONLINE +</STMTTRN> +''' + for amount in ("null", "-null"): + input = input_template.format(amount=amount) + txn = soup_maker(input) + + transaction = OfxParser.parseTransaction(txn.find('stmttrn')) + + self.assertEquals(0, transaction.amount) + + class TestTransaction(TestCase): def testThatAnEmptyTransactionIsValid(self): t = Transaction() @@ -473,7 +584,6 @@ class TestInvestmentAccount(TestCase): # Success! - class TestVanguardInvestmentStatement(TestCase): def testForUnclosedTags(self): ofx = OfxParser.parse(open_file('vanguard.ofx')) @@ -497,6 +607,20 @@ class TestVanguardInvestmentStatement(TestCase): self.assertEquals(len(ofx.security_list), 2) +class TestVanguard401kStatement(TestCase): + def testReadTransfer(self): + ofx = OfxParser.parse(open_file('vanguard401k.ofx')) + self.assertTrue(hasattr(ofx, 'account')) + self.assertTrue(hasattr(ofx.account, 'statement')) + self.assertTrue(hasattr(ofx.account.statement, 'transactions')) + self.assertEquals(len(ofx.account.statement.transactions), 5) + self.assertEquals(ofx.account.statement.transactions[-1].id, + '1234567890123456795AAA') + self.assertEquals('transfer', ofx.account.statement.transactions[-1].type) + self.assertEquals(ofx.account.statement.transactions[-1].inv401ksource, + 'MATCH') + + class TestFidelityInvestmentStatement(TestCase): def testForUnclosedTags(self): ofx = OfxParser.parse(open_file('fidelity.ofx')) @@ -510,6 +634,66 @@ class TestFidelityInvestmentStatement(TestCase): self.assertEquals(len(ofx.security_list), 7) +class Test401InvestmentStatement(TestCase): + def testTransferAggregate(self): + ofx = OfxParser.parse(open_file('investment_401k.ofx')) + expected_txns = [{'id': '1', + 'type': 'buymf', + 'units': Decimal('8.846699'), + 'unit_price': Decimal('22.2908'), + 'total': Decimal('-197.2'), + 'security': 'FOO'}, + {'id': '2', + 'type': 'transfer', + 'units': Decimal('6.800992'), + 'unit_price': Decimal('29.214856'), + 'total': Decimal('0.0'), + 'security': 'BAR'}, + {'id': '3', + 'type': 'transfer', + 'units': Decimal('-9.060702'), + 'unit_price': Decimal('21.928764'), + 'total': Decimal('0.0'), + 'security': 'BAZ'}] + for txn, expected_txn in zip(ofx.account.statement.transactions, expected_txns): + self.assertEquals(txn.id, expected_txn['id']) + self.assertEquals(txn.type, expected_txn['type']) + self.assertEquals(txn.units, expected_txn['units']) + self.assertEquals(txn.unit_price, expected_txn['unit_price']) + self.assertEquals(txn.total, expected_txn['total']) + self.assertEquals(txn.security, expected_txn['security']) + + expected_positions = [{'security': 'FOO', + 'units': Decimal('17.604312'), + 'unit_price': Decimal('22.517211')}, + {'security': 'BAR', + 'units': Decimal('13.550983'), + 'unit_price': Decimal('29.214855')}, + {'security': 'BAZ', + 'units': Decimal('0.0'), + 'unit_price': Decimal('0.0')}] + for pos, expected_pos in zip(ofx.account.statement.positions, expected_positions): + self.assertEquals(pos.security, expected_pos['security']) + self.assertEquals(pos.units, expected_pos['units']) + self.assertEquals(pos.unit_price, expected_pos['unit_price']) + + +class TestSuncorpBankStatement(TestCase): + def testCDATATransactions(self): + ofx = OfxParser.parse(open_file('suncorp.ofx')) + accounts = ofx.accounts + self.assertEquals(len(accounts), 1) + account = accounts[0] + transactions = account.statement.transactions + self.assertEquals(len(transactions), 1) + transaction = transactions[0] + self.assertEquals(transaction.payee, "EFTPOS WDL HANDYWAY ALDI STORE") + self.assertEquals( + transaction.memo, + "EFTPOS WDL HANDYWAY ALDI STORE GEELONG WEST VICAU") + self.assertEquals(transaction.amount, Decimal("-16.85")) + + class TestAccountInfoAggregation(TestCase): def testForFourAccounts(self): ofx = OfxParser.parse(open_file('account_listing_aggregation.ofx')) @@ -596,6 +780,7 @@ class TestGracefulFailures(TestCase): self.assertRaises(OfxParserException, OfxParser.parse, open_file('fail_nice/empty_balance.ofx')) + class TestParseSonrs(TestCase): def testSuccess(self): diff --git a/tests/test_write.py b/tests/test_write.py index 3366217..28361b0 100644 --- a/tests/test_write.py +++ b/tests/test_write.py @@ -6,6 +6,7 @@ import sys sys.path.append('..') from .support import open_file + class TestOfxWrite(TestCase): def test_write(self): test_file = open_file('fidelity.ofx') |