NPO-Accounting/import2ledger Changeset - 8516134687e8

Changeset - 8516134687e8

Parent rev.

Child rev.

[Not reviewed]

0 3 0

Brett Smith - 5 years ago 2019-11-11 15:18:21
brettcsmith@brettcsmith.org

hooks.ledger_entry: Improve date handling.

Check that we have the date field used in the payee line, and not just
'date'.
Allow other date fields to be None since they may not be used by the
template.

3 files changed with 24 insertions and 9 deletions:

import2ledger/hooks/ledger_entry.py

setup.py

tests/test_hook_ledger_entry.py

0 comments (0 inline, 0 general)

import2ledger/hooks/ledger_entry.py

➞

Show inline comments

@@ ... / @@ -131,231 +131,234 @@ class AccountSplitter: @@
     def __init__(self, signed_currencies, signed_currency_fmt, unsigned_currency_fmt,
                  template_name):
         self.splits = []
         self.metadata = []
         self.signed_currency_fmt = signed_currency_fmt
         self.unsigned_currency_fmt = unsigned_currency_fmt
         self.signed_currencies = set(signed_currencies)
         self.template_name = template_name
         self._last_template_vars = object()
     def is_empty(self):
         return not self.splits
     def add(self, account, amount_expr):
         try:
             clean_expr = AmountTokenTransformer.from_str(amount_expr).transform()
             compiled_expr = compile(clean_expr, self.template_name, 'eval')
         except (SyntaxError, tokenize.TokenError, ValueError) as error:
             raise errors.UserInputConfigurationError(error.args[0], amount_expr)
         else:
             self.splits.append((account, compiled_expr))
             self.metadata.append('')
     def set_metadata(self, metadata_s):
         self.metadata[-1] = metadata_s
     def _currency_decimal(self, amount, currency):
         return decimal.Decimal(babel.numbers.format_currency(amount, currency, '###0.###'))
     def _balance_amounts(self, amounts, to_amount):
         cmp_func = operator.lt if to_amount > 0 else operator.gt
         should_balance = functools.partial(cmp_func, 0)
         remainder = to_amount
         balance_index = None
         for index, (_, amount) in enumerate(amounts):
             if should_balance(amount):
                 remainder -= amount
                 balance_index = index
         if balance_index is None:
             pass
         elif (abs(remainder) / abs(to_amount)) >= decimal.Decimal('.1'):
             raise errors.UserInputConfigurationError(
                 "template can't balance amounts to {}".format(to_amount),
                 self.template_name,
+            )
         else:
             account_name, start_amount = amounts[balance_index]
             amounts[balance_index] = (account_name, start_amount + remainder)
     def _build_amounts(self, template_vars):
         try:
             amounts = [
                 (account,
                  self._currency_decimal(eval(amount_expr, self.EVAL_GLOBALS, template_vars),
                                         template_vars['currency']),
                 ) for account, amount_expr in self.splits
+            ]
         except (ArithmeticError, NameError, TypeError, ValueError) as error:
             raise errors.UserInputConfigurationError(
                 "{}: {}".format(type(error).__name__, error),
                 "template {!r}".format(self.template_name)
             ) from error
         if sum(amt for _, amt in amounts) != 0:
             self._balance_amounts(amounts, template_vars['amount'])
             self._balance_amounts(amounts, -template_vars['amount'])
         return amounts
     def _iter_splits(self, template_vars):
         amounts = self._build_amounts(template_vars)
         if template_vars['currency'] in self.signed_currencies:
             amt_fmt = self.signed_currency_fmt
         else:
             amt_fmt = self.unsigned_currency_fmt
         for (account, amount), metadata in zip(amounts, self.metadata):
             if amount == 0:
                 yield ''
             else:
                 account_s = account.format_map(template_vars)
                 amount_s = babel.numbers.format_currency(amount, template_vars['currency'], amt_fmt)
                 sep_len = max(2, self.TARGET_ACCTLINE_LEN - len(account_s) - len(amount_s))
                 yield '\n    {}{}{}{}'.format(
                     account_s, ' ' * sep_len, amount_s,
                     metadata.format_map(template_vars),
+                )
     def render_next(self, template_vars):
         if template_vars is not self._last_template_vars:
             self._split_iter = self._iter_splits(template_vars)
             self._last_template_vars = template_vars
         return next(self._split_iter)
 class Template:
     ACCOUNT_SPLIT_RE = re.compile(r'(?:\t|  )\s*')
     DATE_FMT = '%Y/%m/%d'
     PAYEE_LINE_RE = re.compile(r'\{(\w*_)*date\}')
+    PAYEE_LINE_RE = re.compile(r'^\{(\w*_)*date\}\s')
     SIGNED_CURRENCY_FMT = '¤#,##0.###;¤-#,##0.###'
     UNSIGNED_CURRENCY_FMT = '#,##0.### ¤¤'
     def __init__(self, template_s, signed_currencies=frozenset(),
                  date_fmt=DATE_FMT,
                  signed_currency_fmt=SIGNED_CURRENCY_FMT,
                  unsigned_currency_fmt=UNSIGNED_CURRENCY_FMT,
                  template_name='<template>'):
         self.date_fmt = date_fmt
         self.date_field = 'date'
         self.splitter = AccountSplitter(
             signed_currencies, signed_currency_fmt, unsigned_currency_fmt, template_name)
         lines = self._template_lines(template_s)
         self.format_funcs = []
         try:
             self.format_funcs.append(next(lines).format_map)
         except StopIteration:
             return
         metadata = []
         for line in lines:
             if line.startswith(';'):
                 metadata.append(line)
             else:
                 self._add_str_func(metadata)
                 metadata = []
                 line = line.strip()
                 match = self.ACCOUNT_SPLIT_RE.search(line)
                 if match is None:
                     raise errors.UserInputError("no amount expression found", line)
                 account = line[:match.start()]
                 amount_expr = line[match.end():]
                 self.splitter.add(account, amount_expr)
                 self.format_funcs.append(self.splitter.render_next)
         self._add_str_func(metadata)
         self.format_funcs.append('\n'.format_map)
     def _nonblank_lines(self, s):
         for line in s.splitlines(True):
             line = line.strip()
             if line:
                 yield line
     def _template_lines(self, template_s):
         lines = self._nonblank_lines(template_s)
         try:
             line1 = next(lines)
         except StopIteration:
             return
         if self.PAYEE_LINE_RE.match(line1):
         match = self.PAYEE_LINE_RE.match(line1)
         if match:
             self.date_field = match.group(0)[1:-2]
             yield '\n' + line1
         else:
             yield '\n{date} {payee}'
             yield line1
         yield from lines
     def _add_str_func(self, str_seq):
         str_flat = ''.join('\n      ' + s for s in str_seq)
         if not str_flat:
             pass
         elif self.splitter.is_empty():
             self.format_funcs.append(str_flat.format_map)
         else:
             self.splitter.set_metadata(str_flat)
     def render(self, template_vars):
         # template_vars must have these keys.  Raise a KeyError if not.
         template_vars['currency']
         template_vars['payee']
         try:
             date = template_vars['date']
         except KeyError:
             date = datetime.date.today()
         if template_vars.get(self.date_field) is None:
             raise errors.UserInputConfigurationError(
                 "entry needs {} field but that's not set by the importer".format(
                     self.date_field,
                 ), self.splitter.template_name)
         render_vars = {
             'amount': strparse.currency_decimal(template_vars['amount']),
             'date': date.strftime(self.date_fmt),
+        }
         for key, value in template_vars.items():
             if key.endswith('_date'):
+            if value is not None and (key == 'date' or key.endswith('_date')):
                 render_vars[key] = value.strftime(self.date_fmt)
         all_vars = collections.ChainMap(render_vars, template_vars)
         return ''.join(f(all_vars) for f in self.format_funcs)
     def is_empty(self):
         return not self.format_funcs
 class LedgerEntryHook:
     KIND = HOOK_KINDS.OUTPUT
     def __init__(self, config):
         self.config = config
     @staticmethod
     @functools.lru_cache()
     def _load_template(config, section_name, config_key):
         section_config = config.get_section(section_name)
         try:
             template_s = section_config[config_key]
         except KeyError:
             raise errors.UserInputConfigurationError(
                 "Ledger template not defined in [{}]".format(section_name),
                 config_key,
+            )
         return Template(
             template_s,
             date_fmt=section_config['date_format'],
             signed_currencies=[code.strip().upper() for code in section_config['signed_currencies'].split(',')],
             signed_currency_fmt=section_config['signed_currency_format'],
             unsigned_currency_fmt=section_config['unsigned_currency_format'],
             template_name=config_key,
+        )
     def run(self, entry_data):
         try:
             template_key = entry_data['ledger template']
         except KeyError:
             template_key = '{} {} ledger entry'.format(
                 strparse.rslice_words(entry_data['importer_module'], -1, '.', 1),
                 entry_data['importer_class'][:-8].lower(),
+            )
         try:
             template = self._load_template(self.config, None, template_key)
         except errors.UserInputConfigurationError as error:
             if error.strerror.startswith('Ledger template not defined '):
                 have_template = False
             else:
                 raise
         else:
             have_template = not template.is_empty()
         if not have_template:
             logger.warning("no Ledger template defined as %r", template_key)
         else:
             with self.config.open_output_file() as out_file:
                 print(template.render(entry_data), file=out_file, end='')

setup.py

➞

Show inline comments

 #!/usr/bin/env python3
 import sys
 from setuptools import setup, find_packages
 REQUIREMENTS = {
     'install_requires': [
         'babel',
         'enum34;python_version<"3.4"',
     ],
     'setup_requires': ['pytest-runner'],
     'extras_require': {
         'brightfunds': ['xlrd'],
         'nbpy2017': ['beautifulsoup4', 'html5lib'],
     },
+}
 all_extras_require = [
     req for reqlist in REQUIREMENTS['extras_require'].values() for req in reqlist
+]
 REQUIREMENTS['extras_require']['all_importers'] = all_extras_require
 REQUIREMENTS['tests_require'] = [
     'pytest',
     'PyYAML',
     *all_extras_require,
+]
 setup(
     name='import2ledger',
     description="Import different sources of financial data to Ledger",
-    version='0.10.0',
+    version='0.10.1',
     author='Brett Smith',
     author_email='brettcsmith@brettcsmith.org',
     license='GNU AGPLv3+',
     packages=find_packages(include=['import2ledger', 'import2ledger.*']),
     entry_points={
         'console_scripts': ['import2ledger = import2ledger.__main__:main'],
     },
     **REQUIREMENTS,
+)

tests/test_hook_ledger_entry.py

➞

Show inline comments

@@ ... / @@ -89,192 +89,204 @@ def test_complex_template(): @@
         "  ;Program: Spectrum Defense",
         "  ;Entity: T-T",
         "  Income:Donations:General  $-5.65",
         "  ;Entity: T-T",
+    ]
 def test_balancing():
     lines = render_lines(template_vars('FF', '1.01'), 'FiftyFifty')
     assert lines == [
         "",
         "2015/03/14 FF",
         "  Accrued:Accounts Receivable  1.01 USD",
         "  Income:Donations  -0.50 USD",
         "  Income:Sales  -0.51 USD",
+    ]
 def test_multivalue():
     render_vars = template_vars('DD', '150.00', other_vars={
         'tax': decimal.Decimal('12.50'),
     })
     lines = render_lines(render_vars, 'Multivalue')
     assert lines == [
         "",
         "2015/03/14 DD",
         "  Expenses:Taxes  12.50 USD",
         "  ;TaxAuthority: IRS",
         "  Accrued:Accounts Receivable  137.50 USD",
         "  Income:RBI  -15.00 USD",
         "  Income:Donations  -135.00 USD",
+    ]
 def test_zeroed_account_skipped():
     render_vars = template_vars('GG', '110.00', other_vars={
         'tax': decimal.Decimal(0),
     })
     lines = render_lines(render_vars, 'Multivalue')
     assert lines == [
         "",
         "2015/03/14 GG",
         "  Accrued:Accounts Receivable  110.00 USD",
         "  Income:RBI  -11.00 USD",
         "  Income:Donations  -99.00 USD",
+    ]
 def test_zeroed_account_last():
     render_vars = template_vars('JJ', '90.00', other_vars={
         'item_sales': decimal.Decimal(0),
     })
     lines = render_lines(render_vars, 'Multisplit')
     assert lines == [
         "",
         "2015/03/14 JJ",
         "  Assets:Cash  90.00 USD",
         "  Income:Sales  -90.00 USD",
         "  ; :NonItem:",
+    ]
 def test_multiple_postings_same_account():
     render_vars = template_vars('LL', '80.00', other_vars={
         'item_sales': decimal.Decimal(30),
     })
     lines = render_lines(render_vars, 'Multisplit')
     assert lines == [
         "",
         "2015/03/14 LL",
         "  Assets:Cash  80.00 USD",
         "  Income:Sales  -50.00 USD",
         "  ; :NonItem:",
         "  Income:Sales  -30.00 USD",
         "  ; :Item:",
+    ]
 def test_custom_payee_line():
     render_vars = template_vars('ZZ', '10.00', other_vars={
         'custom_date': datetime.date(2014, 2, 13),
     })
     lines = render_lines(render_vars, 'Custom Payee')
     assert lines == [
         "",
         "2014/02/13  ZZ - Custom",
         "  Accrued:Accounts Receivable  10.00 USD",
         "  Income:Donations  -10.00 USD",
+    ]
 def test_line1_not_custom_payee():
     render_vars = template_vars('VV', '15.00', other_vars={
         'custom_date': datetime.date(2014, 2, 12),
     })
     lines = render_lines(render_vars, 'Simplest')
     assert lines == [
         "",
         "2015/03/14 VV",
         "  Accrued:Accounts Receivable  15.00 USD",
         "  Income:Donations  -15.00 USD",
+    ]
 def test_custom_date_missing():
     render_vars = template_vars('YY', '20.00')
     with pytest.raises(errors.UserInputConfigurationError):
         render_lines(render_vars, 'Custom Payee')
 def test_custom_date_is_none():
     render_vars = template_vars('YZ', '25.00', other_vars={
         'custom_date': None,
     })
     with pytest.raises(errors.UserInputConfigurationError):
         render_lines(render_vars, 'Custom Payee')
 @pytest.mark.parametrize('amount,expect_fee', [
     (40, 3),
     (80, 6),
 ])
 def test_conditional(amount, expect_fee):
     expect_cash = amount - expect_fee
     amount_s = '{:.02f}'.format(amount)
     render_vars = template_vars('Buyer', amount_s)
     lines = render_lines(render_vars, 'Conditional')
     assert lines == [
         "",
         "2015/03/14 Buyer",
         "  Assets:Cash  {:.02f} USD".format(expect_cash),
         "  Expenses:Banking Fees  {:.02f} USD".format(expect_fee),
         "  Income:Sales  -{} USD".format(amount_s),
+    ]
 def test_string_conditionals():
     render_vars = template_vars('MM', '6', other_vars={
         'false': '',
         'true': 'true',
     })
     lines = render_lines(render_vars, 'StringConditional')
     assert lines == [
         "",
         "2015/03/14 MM",
         "  Income:Sales  -1.00 USD",
         "  Income:Sales  -2.00 USD",
         "  Income:Sales  -3.00 USD",
         "  Assets:Cash  6.00 USD",
+    ]
 def test_self_balanced():
     # The amount needs to be different than what's in the template.
     render_vars = template_vars('NN', '0')
     lines = render_lines(render_vars, 'SelfBalanced')
     assert lines == [
         "",
         "2015/03/14 NN",
         "  Income:Sales  -5.00 USD",
         "  Assets:Cash  5.00 USD",
+    ]
 @pytest.mark.parametrize('amount_expr', [
     '',
     'name',
     '-',
     '()',
     '+()',
     '{}',
     '{{}}',
     '{()}',
     '{name',
     'name}',
     '{42}',
     '(5).real',
     '{amount.real}',
     '{amount.is_nan()}',
     '{Decimal}',
     '{FOO}',
 ])
 def test_bad_amount_expression(amount_expr):
     with pytest.raises(errors.UserInputError):
         ledger_entry.Template(" Income  " + amount_expr)
 class Config:
     def __init__(self, use_section):
         self.section_name = use_section
         self.stdout = io.StringIO()
     @contextlib.contextmanager
     def open_output_file(self):
         yield self.stdout
     def get_section(self, name=None):
         return config[self.section_name]
 def run_hook(entry_data, config_section):
     hook_config = Config(config_section)
     hook = ledger_entry.LedgerEntryHook(hook_config)
     assert hook.run(entry_data) is None
     stdout = hook_config.stdout.getvalue()
     return normalize_whitespace(stdout).splitlines()
 def test_hook_renders_template():
     entry_data = template_vars('BB', '0.99')
     lines = run_hook(entry_data, 'Simplest')
     assert lines == [
         "",
         "2015-03-14 BB",
         "  Accrued:Accounts Receivable  $0.99",
         "  Income:Donations  -$0.99",
+    ]
 def test_hook_handles_empty_template():

0 comments (0 inline, 0 general)