NPO-Accounting/conservancy_beancount Changeset - fdd9f2847b78

Changeset - fdd9f2847b78

Parent rev.

Child rev.

[Not reviewed]

0 10 0

Brett Smith - 3 years ago 2021-01-29 14:38:37
brettcsmith@brettcsmith.org

plugin: Skip enum value checks with a flag+FIXME.

We've long supported skipping documentation checks by flagging the
transaction. We haven't done the same for enumerated metadata because we
need it less often, and bad values tend to do more damage to reports.

However, occasionally when something very off-process happens, we do need it
as a matter of expediency. So support it.

In order to skip validation of these fields, the plugin requires that the
value start with the string "FIXME". This helps ensure that reports have a
consistent way to detect and warn about unfilled values in flagged
transactions.

10 files changed with 71 insertions and 10 deletions:

conservancy_beancount/plugin/core.py

conservancy_beancount/plugin/meta_paypal_id.py

conservancy_beancount/plugin/meta_tax_implication.py

setup.py

tests/test_meta_expense_type.py

tests/test_meta_income_type.py

tests/test_meta_paypal_id.py

tests/test_meta_project.py

tests/test_meta_tax_implication.py

tests/testutil.py

0 comments (0 inline, 0 general)

conservancy_beancount/plugin/core.py

➞

Show inline comments

@@ ... / @@ -139,96 +139,104 @@ class MetadataEnum: @@
 class TransactionHook(Hook[Transaction]):
     DIRECTIVE = Transaction
     SKIP_FLAGS: Container[str] = frozenset()
     TXN_DATE_RANGE = ranges.DateRange(DEFAULT_START_DATE, DEFAULT_STOP_DATE)
     def _run_on_txn(self, txn: Transaction) -> bool:
         """Check whether we should run on a given transaction
         This method implements our usual checks for whether or not a hook
         should run on a given transaction. It's here for subclasses to use in
         their own implementations. See _PostingHook below for an example.
         """
         return (
             txn.flag not in self.SKIP_FLAGS
             and txn.date in self.TXN_DATE_RANGE
             and not data.is_opening_balance_txn(txn)
+        )
 class _PostingHook(TransactionHook, metaclass=abc.ABCMeta):
     def __init_subclass__(cls) -> None:
         cls.HOOK_GROUPS = cls.HOOK_GROUPS.union(['posting'])
     def _is_flagged_fixme(self, post: data.Posting, value: MetaValue) -> bool:
         return (
             post.meta.txn.flag == '!'
             and isinstance(value, str)
             and value.startswith('FIXME')
+        )
     def _run_on_post(self, txn: Transaction, post: data.Posting) -> bool:
         return True
     def run(self, txn: Transaction) -> errormod.Iter:
         if self._run_on_txn(txn):
             for post in data.Posting.from_txn(txn):
                 if self._run_on_post(txn, post):
                     yield from self.post_run(txn, post)
     @abc.abstractmethod
     def post_run(self, txn: Transaction, post: data.Posting) -> errormod.Iter: ...
 class _NormalizePostingMetadataHook(_PostingHook):
     """Base class to normalize posting metadata from an enum."""
     # This class provides basic functionality to filter postings, normalize
     # metadata values, and set default values.
     METADATA_KEY: MetaKey
     VALUES_ENUM: MetadataEnum
     def __init_subclass__(cls) -> None:
         super().__init_subclass__()
         cls.METADATA_KEY = cls.VALUES_ENUM.key
         cls.HOOK_GROUPS = cls.HOOK_GROUPS.union(['metadata', cls.METADATA_KEY])
     # If the posting does not specify METADATA_KEY, the hook calls
     # _default_value to get a default. This method should either return
     # a value string from METADATA_ENUM, or else raise InvalidMetadataError.
     # This base implementation does the latter.
     def _default_value(self, txn: Transaction, post: data.Posting) -> MetaValueEnum:
         raise errormod.InvalidMetadataError(txn, self.METADATA_KEY, None, post)
     def post_run(self, txn: Transaction, post: data.Posting) -> errormod.Iter:
         source_value = post.meta.get(self.METADATA_KEY)
         set_value = source_value
         error: Optional[errormod.Error] = None
         if source_value is None:
             try:
                 set_value = self._default_value(txn, post)
             except errormod.Error as error_:
                 error = error_
         else:
             try:
                 set_value = self.VALUES_ENUM[source_value]
             except KeyError:
                 error = errormod.InvalidMetadataError(
                     txn, self.METADATA_KEY, source_value, post,
+                )
                 if not self._is_flagged_fixme(post, source_value):
                     error = errormod.InvalidMetadataError(
                         txn, self.METADATA_KEY, source_value, post,
+                    )
         if error is None:
             post.meta[self.METADATA_KEY] = set_value
         else:
             yield error
 class _RequireLinksPostingMetadataHook(_PostingHook):
     """Base class to require that posting metadata include links"""
     # This base class confirms that a posting's metadata has one or more links
     # under one of the metadata keys listed in CHECKED_METADATA.
     # Most subclasses only need to define CHECKED_METADATA and _run_on_post.
     CHECKED_METADATA: Sequence[MetaKey]
     def __init_subclass__(cls) -> None:
         super().__init_subclass__()
         cls.HOOK_GROUPS = cls.HOOK_GROUPS.union(cls.CHECKED_METADATA).union('metadata')
     def _check_metadata(self,
                         txn: Transaction,
                         post: data.Posting,
                         keys: Sequence[MetaKey],
     ) -> Iterator[errormod.InvalidMetadataError]:
         have_docs = False
         for key in keys:

conservancy_beancount/plugin/meta_paypal_id.py

➞

Show inline comments

@@ ... / @@ -18,26 +18,26 @@ class MetaPayPalID(core._PostingHook): @@
     METADATA_KEY = 'paypal-id'
     HOOK_GROUPS = frozenset(['metadata', METADATA_KEY])
     TXN_ID_RE = re.compile(r'^[A-Z0-9]{17}$')
     INVOICE_ID_RE = re.compile(r'^INV2(?:-[A-Z0-9]{4}){4}$')
     def _run_on_post(self, txn: Transaction, post: data.Posting) -> bool:
         if post.account.is_under('Assets:PayPal'):
             return True
         elif post.account.is_under('Assets:Receivable'):
             return self.METADATA_KEY in post.meta
         else:
             return False
     def post_run(self, txn: Transaction, post: data.Posting) -> errormod.Iter:
         if post.account.is_under('Assets:Receivable'):
             regexp = self.INVOICE_ID_RE
         else:
             regexp = self.TXN_ID_RE
         value = post.meta.get(self.METADATA_KEY)
         try:
             # A bad argument type is okay because we catch the TypeError.
             match = regexp.match(value)  # type:ignore[arg-type]
         except TypeError:
             match = None
         if match is None:
+        if match is None and not self._is_flagged_fixme(post, value):
             yield errormod.InvalidMetadataError(txn, self.METADATA_KEY, value, post)

conservancy_beancount/plugin/meta_tax_implication.py

➞

Show inline comments

@@ ... @@
         '1099-MISC-Other',
         '1099-NEC',
         'Bank-Transfer',
         'Chargeback',
         'Foreign-Corporation',
         'Foreign-Grantee',
         'Foreign-Individual-Contractor',
         'Loan',
         'Refund',
         'Reimbursement',
         'Retirement-Pretax',
         'Tax-Payment',
         'USA-501c3',
         'USA-Corporation',
         'USA-Grantee',
         'W2',
+    ]
     _ALIASES = dict(
         alias for value in _STDNAMES for alias in _make_aliases(value)
+    )
     _ALIASES['1099'] = '1099-NEC'
     VALUES_ENUM = core.MetadataEnum('tax-implication', _STDNAMES, _ALIASES)
     del _STDNAMES, _ALIASES
     # Sometimes we accrue a payment before we have determined the recipient's
     # tax status.
     SKIP_FLAGS = '!'
     def __init__(self, config: configmod.Config) -> None:
         self.payment_threshold = -config.payment_threshold()
     def _run_on_post(self, txn: Transaction, post: data.Posting) -> bool:
         return (
             post.account.is_cash_equivalent()
             and post.units.number < self.payment_threshold
+        )

setup.py

➞

Show inline comments

 #!/usr/bin/env python3
 from setuptools import setup
 setup(
     name='conservancy_beancount',
     description="Plugin, library, and reports for reading Conservancy's books",
-    version='1.16.1',
+    version='1.16.2',
     author='Software Freedom Conservancy',
     author_email='info@sfconservancy.org',
     license='GNU AGPLv3+',
     install_requires=[
         'babel>=2.6',  # Debian:python3-babel
         'beancount>=2.2',  # Debian:beancount
         'GitPython>=2.0',  # Debian:python3-git
         # 1.4.1 crashes when trying to save some documents.
         'odfpy>=1.4.0,!=1.4.1',  # Debian:python3-odf
         'pdfminer.six>=20200101',
         'PyYAML>=3.0',  # Debian:python3-yaml
         'regex',  # Debian:python3-regex
         'rt>=2.0',
     ],
     setup_requires=[
         'pytest-mypy',
         'pytest-runner',  # Debian:python3-pytest-runner
     ],
     tests_require=[
         'mypy>=0.770',  # Debian:python3-mypy
         'pytest',  # Debian:python3-pytest
     ],

tests/test_meta_expense_type.py

➞

Show inline comments

@@ ... @@
     (testutil.FUTURE_DATE, 'program'),
     (testutil.FY_START_DATE, 'program'),
     (testutil.FY_MID_DATE, 'program'),
     (testutil.PAST_DATE, None),
 ])
 def test_default_value_set_in_date_range(hook, date, set_value):
     txn = testutil.Transaction(date=date, postings=[
         ('Liabilites:CreditCard', -25),
         ('Expenses:General', 25),
     ])
     errors = list(hook.run(txn))
     assert not errors
     expect_meta = None if set_value is None else {TEST_KEY: set_value}
     testutil.check_post_meta(txn, None, expect_meta)
 @pytest.mark.parametrize('src_value', INVALID_VALUES)
 def test_flagged_txn_checked(hook, src_value):
     txn = testutil.Transaction(flag='!', postings=[
         ('Assets:Cash', -25),
         ('Expenses:General', 25, {TEST_KEY: src_value}),
     ])
     errors = list(hook.run(txn))
     assert errors
     testutil.check_post_meta(txn, None, {TEST_KEY: src_value})
 @pytest.mark.parametrize('src_value', testutil.FIXME_VALUES)
 def test_flagged_fixme_ok(hook, src_value):
     txn = testutil.Transaction(flag='!', postings=[
         ('Assets:Cash', -25),
         ('Expenses:General', 25, {TEST_KEY: src_value}),
     ])
     errors = list(hook.run(txn))
     assert not errors
     testutil.check_post_meta(txn, None, {TEST_KEY: src_value})

tests/test_meta_income_type.py

➞

Show inline comments

@@ ... / @@ -130,24 +130,34 @@ def test_no_default_value(hook, account): @@
     (testutil.FUTURE_DATE, 'Donations'),
     (testutil.FY_START_DATE, 'Donations'),
     (testutil.FY_MID_DATE, 'Donations'),
     (testutil.PAST_DATE, None),
 ])
 def test_default_value_set_in_date_range(hook, date, set_value):
     txn = testutil.Transaction(date=date, postings=[
         ('Assets:Cash', 25),
         ('Income:Donations', -25),
     ])
     errors = list(hook.run(txn))
     assert not errors
     expect_meta = None if set_value is None else {TEST_KEY: set_value}
     testutil.check_post_meta(txn, None, expect_meta)
 @pytest.mark.parametrize('src_value', INVALID_VALUES)
 def test_flagged_txn_checked(hook, src_value):
     txn = testutil.Transaction(flag='!', postings=[
         ('Assets:Cash', 25),
         ('Income:Other', -25, {TEST_KEY: src_value}),
     ])
     errors = list(hook.run(txn))
     assert errors
     testutil.check_post_meta(txn, None, {TEST_KEY: src_value})
 @pytest.mark.parametrize('src_value', testutil.FIXME_VALUES)
 def test_flagged_fixme_ok(hook, src_value):
     txn = testutil.Transaction(flag='!', postings=[
         ('Assets:Cash', 25),
         ('Income:Other', -25, {TEST_KEY: src_value}),
     ])
     errors = list(hook.run(txn))
     assert not errors
     testutil.check_post_meta(txn, None, {TEST_KEY: src_value})

tests/test_meta_paypal_id.py

➞

Show inline comments

@@ ... / @@ -171,24 +171,36 @@ def test_required_for_assets_paypal(hook): @@
     VALID_TXN_IDS,
     VALID_INVOICE_IDS,
 ))
 def test_invoice_payment_transaction_ok(hook, txn_id, inv_id):
     txn = testutil.Transaction(**{TEST_KEY: txn_id}, postings=[
         ('Assets:Receivable:Accounts', -100, {TEST_KEY: inv_id}),
         ('Assets:PayPal', 97),
         ('Expenses:BankingFees', 3),
     ])
     assert not list(hook.run(txn))
 def test_not_required_on_opening(hook):
     txn = testutil.Transaction(postings=[
         ('Assets:PayPal', 1000),
         (next(testutil.OPENING_EQUITY_ACCOUNTS), -1000),
     ])
     assert not list(hook.run(txn))
 def test_still_required_on_flagged_txn(hook):
     txn = testutil.Transaction(flag='!', postings=[
         ('Assets:PayPal', 1000),
         ('Income:Donations', -1000),
     ])
     assert list(hook.run(txn))
 @pytest.mark.parametrize('account,src_value', testutil.combine_values(
     ACCOUNTS,
     testutil.FIXME_VALUES,
 ))
 def test_flagged_fixme_ok(hook, account, src_value):
     txn = testutil.Transaction(flag='!', postings=[
         (account, 200, {TEST_KEY: src_value}),
         ('Income:Donations', -200),
     ])
     assert not list(hook.run(txn))
     testutil.check_post_meta(txn, {TEST_KEY: src_value}, None)

tests/test_meta_project.py

➞

Show inline comments

@@ ... / @@ -180,24 +180,34 @@ def test_missing_project_data(repo_path): @@
 def test_invalid_project_data(repo_path_s, data_path_s):
     config = testutil.TestConfig(repo_path=repo_path_s)
     with pytest.raises(errormod.ConfigurationError):
         meta_project.MetaProject(config, Path(data_path_s))
 def test_not_required_on_opening(hook):
     txn = testutil.OpeningBalance('Equity:Funds:Unrestricted')
     assert not list(hook.run(txn))
 def test_always_required_on_restricted_funds(hook):
     acct = 'Equity:Funds:Restricted'
     txn = testutil.OpeningBalance(acct)
     actual = {error.message for error in hook.run(txn)}
     assert actual == {f'{acct} missing project'}
 @pytest.mark.parametrize('src_value', INVALID_VALUES)
 def test_still_required_on_flagged_txn(hook, src_value):
     txn = testutil.Transaction(flag='!', **{TEST_KEY: src_value}, postings=[
         ('Assets:Cash', -25),
         ('Expenses:General', 25),
     ])
     errors = list(hook.run(txn))
     assert errors
     testutil.check_post_meta(txn, None, None)
 @pytest.mark.parametrize('src_value', testutil.FIXME_VALUES)
 def test_flagged_fixme_ok(hook, src_value):
     txn = testutil.Transaction(flag='!', postings=[
         ('Assets:Cash', -25),
         ('Expenses:General', 25, {TEST_KEY: src_value}),
     ])
     errors = list(hook.run(txn))
     assert not errors
     testutil.check_post_meta(txn, None, {TEST_KEY: src_value})

tests/test_meta_tax_implication.py

➞

Show inline comments

@@ ... / @@ -116,30 +116,39 @@ def test_asset_credits_skipped(hook): @@
         ('Income:Donations', -25),
         ('Assets:Cash', 25),
     ])
     errors = list(hook.run(txn))
     assert not errors
     testutil.check_post_meta(txn, None, None)
 @pytest.mark.parametrize('date,need_value', [
     (testutil.EXTREME_FUTURE_DATE, False),
     (testutil.FUTURE_DATE, True),
     (testutil.FY_START_DATE, True),
     (testutil.FY_MID_DATE, True),
     (testutil.PAST_DATE, False),
 ])
 def test_validation_only_in_date_range(hook, date, need_value):
     txn = testutil.Transaction(date=date, postings=[
         ('Liabilites:CreditCard', 25),
         ('Assets:Cash', -25),
     ])
     errors = list(hook.run(txn))
     assert bool(errors) == bool(need_value)
     testutil.check_post_meta(txn, None, None)
 @pytest.mark.parametrize('src_value', INVALID_VALUES)
-def test_flagged_txn_skipped(hook, src_value):
+def test_flagged_txn_checked(hook, src_value):
     txn = testutil.Transaction(flag='!', **{TEST_KEY: src_value}, postings=[
         ('Liabilities:Payable:Accounts', 25),
         ('Assets:Cash', -25),
     ])
     assert list(hook.run(txn))
 @pytest.mark.parametrize('src_value', testutil.FIXME_VALUES)
 def test_flagged_fixme_ok(hook, src_value):
     txn = testutil.Transaction(flag='!', postings=[
         ('Liabilities:Payable:Accounts', 25),
         ('Assets:Cash', -25, {TEST_KEY: src_value}),
     ])
     assert not list(hook.run(txn))
     testutil.check_post_meta(txn, None, {TEST_KEY: src_value})

tests/testutil.py

➞

Show inline comments

@@ ... / @@ -173,48 +173,54 @@ def Transaction(date=FY_MID_DATE, flag='*', payee=None, @@
         set(tags or ''),
         set(links or ''),
         real_postings,
+    )
 LINK_METADATA_STRINGS = {
     'Invoices/304321.pdf',
     'rt:123/456',
     'rt://ticket/234',
+}
 NON_LINK_METADATA_STRINGS = {
     '',
     ' ',
     '     ',
+}
 NON_STRING_METADATA_VALUES = [
     Decimal(5),
     FY_MID_DATE,
     Amount(50),
     Amount(500, None),
+]
 FIXME_VALUES = [
     'FIXME',
     'FIXME loose comment',
     'FIXME: comment with punctuation',
+]
 OPENING_EQUITY_ACCOUNTS = itertools.cycle([
     'Equity:Funds:Unrestricted',
     'Equity:Funds:Restricted',
     'Equity:OpeningBalance',
 ])
 class ODSCell:
     @classmethod
     def from_row(cls, row):
         return row.getElementsByType(odf.table.TableCell)
     @classmethod
     def from_sheet(cls, spreadsheet):
         for row in spreadsheet.getElementsByType(odf.table.TableRow):
             yield list(cls.from_row(row))
     @classmethod
     def from_ods_file(cls, path):
         ods = odf.opendocument.load(path)
         return cls.from_sheet(ods.spreadsheet)
 def OpeningBalance(acct=None, **txn_meta):
     if acct is None:

0 comments (0 inline, 0 general)