Changeset - e22e63dcca78
[Not reviewed]
0 3 0
Brett Smith - 4 years ago 2020-06-05 14:54:35
brettcsmith@brettcsmith.org
accrual: Make accruals consistent by entity on the accrual side.

It is more common than I realized that we split an invoice by
entity on the accrual side, so this supports that better.

It still disregards inconsistency between accrual entity and payment entity
for reporting purposes, to help keep reporting clean around automatic
imports.

The changes to BaseReport._report shook out because at this point, the group
key is effectively arbitrary and shouldn't be used for any reporting
purposes.
3 files changed with 81 insertions and 40 deletions:
0 comments (0 inline, 0 general)
conservancy_beancount/reports/accrual.py
Show inline comments
...
 
@@ -237,38 +237,44 @@ class AccrualPostings(core.RelatedPostings):
 
    ) -> FrozenSet[MetaValue]:
 
        return frozenset(
 
            post.meta.get('entity') or default
 
            for post in self if pred(post)
 
        )
 

	
 
    def entities(self) -> Iterator[MetaValue]:
 
        yield from self.accrued_entities
 
        yield from self.paid_entities.difference(self.accrued_entities)
 

	
 
    def make_consistent(self) -> Iterator[Tuple[MetaValue, 'AccrualPostings']]:
 
        account_ok = isinstance(self.account, str)
 
        if len(self.accrued_entities) == 1:
 
            entity = next(iter(self.accrued_entities))
 
        else:
 
            entity = None
 
        # `'/' in self.invoice` is just our heuristic to ensure that the
 
        # invoice metadata is "unique enough," and not just a placeholder
 
        # value like "FIXME". It can be refined if needed.
 
        invoice_ok = isinstance(self.invoice, str) and '/' in self.invoice
 
        if account_ok and invoice_ok:
 
        if account_ok and entity is not None and invoice_ok:
 
            yield (self.invoice, self)
 
            return
 
        groups = collections.defaultdict(list)
 
        for post in self:
 
            if invoice_ok:
 
                key = f'{self.invoice} {post.account}'
 
            else:
 
                key = f'{post.account} {post.meta.get("entity")} {post.meta.get("invoice")}'
 
            groups[key].append(post)
 
            post_invoice = self.invoice if invoice_ok else (
 
                post.meta.get('invoice') or 'BlankInvoice'
 
            )
 
            post_entity = entity if entity is not None else (
 
                post.meta.get('entity') or 'BlankEntity'
 
            )
 
            groups[f'{post.account} {post_invoice} {post_entity}'].append(post)
 
        type_self = type(self)
 
        for group_key, posts in groups.items():
 
            yield group_key, type_self(posts, _can_own=True)
 

	
 
    def report_inconsistencies(self) -> Iterable[Error]:
 
        for field_name, get_func in self._FIELDS.items():
 
            if getattr(self, field_name) is self.INCONSISTENT:
 
                for post in self:
 
                    errmsg = 'inconsistent {} for invoice {}: {}'.format(
 
                        field_name.replace('_', '-'),
 
                        self.invoice or "<none>",
 
                        get_func(post),
...
 
@@ -305,34 +311,30 @@ class AccrualPostings(core.RelatedPostings):
 
        try:
 
            empty = start_index == index
 
        except NameError:
 
            empty = True
 
        return self if empty else self[start_index + 1:]
 

	
 

	
 
class BaseReport:
 
    def __init__(self, out_file: TextIO) -> None:
 
        self.out_file = out_file
 
        self.logger = logger.getChild(type(self).__name__)
 

	
 
    def _report(self,
 
                invoice: str,
 
                posts: AccrualPostings,
 
                index: int,
 
    ) -> Iterable[str]:
 
    def _report(self, posts: AccrualPostings, index: int) -> Iterable[str]:
 
        raise NotImplementedError("BaseReport._report")
 

	
 
    def run(self, groups: PostGroups) -> None:
 
        for index, invoice in enumerate(groups):
 
            for line in self._report(str(invoice), groups[invoice], index):
 
            for line in self._report(groups[invoice], index):
 
                print(line, file=self.out_file)
 

	
 

	
 
class AgingODS(core.BaseODS[AccrualPostings, Optional[data.Account]]):
 
    COLUMNS = [
 
        'Date',
 
        'Entity',
 
        'Invoice Amount',
 
        'Booked Amount',
 
        'Ticket',
 
        'Invoice',
 
    ]
...
 
@@ -481,72 +483,64 @@ class AgingReport(BaseReport):
 
    def run(self, groups: PostGroups) -> None:
 
        rows = list(group for group in groups.values() if not group.is_zero())
 
        rows.sort(key=lambda related: (
 
            related.account,
 
            related[0].meta.date,
 
            min(related.entities()) if related.accrued_entities else '',
 
        ))
 
        self.ods.write(rows)
 
        self.ods.save_file(self.out_bin)
 

	
 

	
 
class BalanceReport(BaseReport):
 
    def _report(self,
 
                invoice: str,
 
                posts: AccrualPostings,
 
                index: int,
 
    ) -> Iterable[str]:
 
    def _report(self, posts: AccrualPostings, index: int) -> Iterable[str]:
 
        posts = posts.since_last_nonzero()
 
        date_s = posts[0].meta.date.strftime('%Y-%m-%d')
 
        if index:
 
            yield ""
 
        yield f"{invoice}:"
 
        yield f"{posts.invoice}:"
 
        yield f"  {posts.balance_at_cost()} outstanding since {date_s}"
 

	
 

	
 
class OutgoingReport(BaseReport):
 
    def __init__(self, rt_client: rt.Rt, out_file: TextIO) -> None:
 
        super().__init__(out_file)
 
        self.rt_client = rt_client
 
        self.rt_wrapper = rtutil.RT(rt_client)
 

	
 
    def _primary_rt_id(self, posts: AccrualPostings) -> rtutil.TicketAttachmentIds:
 
        rt_ids = posts.all_meta_links('rt-id')
 
        rt_ids_count = len(rt_ids)
 
        if rt_ids_count != 1:
 
            raise ValueError(f"{rt_ids_count} rt-id links found")
 
        parsed = rtutil.RT.parse(rt_ids.pop())
 
        if parsed is None:
 
            raise ValueError("rt-id is not a valid RT reference")
 
        else:
 
            return parsed
 

	
 
    def _report(self,
 
                invoice: str,
 
                posts: AccrualPostings,
 
                index: int,
 
    ) -> Iterable[str]:
 
    def _report(self, posts: AccrualPostings, index: int) -> Iterable[str]:
 
        posts = posts.since_last_nonzero()
 
        try:
 
            ticket_id, _ = self._primary_rt_id(posts)
 
            ticket = self.rt_client.get_ticket(ticket_id)
 
            # Note we only use this when ticket is None.
 
            errmsg = f"ticket {ticket_id} not found"
 
        except (ValueError, rt.RtError) as error:
 
            ticket = None
 
            errmsg = error.args[0]
 
        if ticket is None:
 
            self.logger.error(
 
                "can't generate outgoings report for %s because no RT ticket available: %s",
 
                invoice, errmsg,
 
                posts.invoice, errmsg,
 
            )
 
            return
 

	
 
        try:
 
            rt_requestor = self.rt_client.get_user(ticket['Requestors'][0])
 
        except (IndexError, rt.RtError):
 
            rt_requestor = None
 
        if rt_requestor is None:
 
            requestor = ''
 
            requestor_name = ''
 
        else:
 
            requestor_name = (
setup.py
Show inline comments
 
#!/usr/bin/env python3
 

	
 
from setuptools import setup
 

	
 
setup(
 
    name='conservancy_beancount',
 
    description="Plugin, library, and reports for reading Conservancy's books",
 
    version='1.1.2',
 
    version='1.1.3',
 
    author='Software Freedom Conservancy',
 
    author_email='info@sfconservancy.org',
 
    license='GNU AGPLv3+',
 

	
 
    install_requires=[
 
        'babel>=2.6',  # Debian:python3-babel
 
        'beancount>=2.2',  # Debian:beancount
 
        # 1.4.1 crashes when trying to save some documents.
 
        'odfpy>=1.4.0,!=1.4.1',  # Debian:python3-odf
 
        'PyYAML>=3.0',  # Debian:python3-yaml
 
        'regex',  # Debian:python3-regex
 
        'rt>=2.0',
tests/test_reports_accrual.py
Show inline comments
...
 
@@ -11,24 +11,25 @@
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU Affero General Public License for more details.
 
#
 
# You should have received a copy of the GNU Affero General Public License
 
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 

	
 
import collections
 
import copy
 
import datetime
 
import io
 
import itertools
 
import logging
 
import operator
 
import re
 

	
 
import babel.numbers
 
import odf.opendocument
 
import odf.table
 
import odf.text
 

	
 
import pytest
 

	
 
from . import testutil
 

	
 
from decimal import Decimal
...
 
@@ -469,84 +470,119 @@ def test_consistency_check_cost():
 
    txn = testutil.Transaction(postings=[
 
        (account, 100, 'EUR', ('1.1251', 'USD'), {'invoice': invoice, 'lineno': 1}),
 
        (account, -100, 'EUR', ('1.125', 'USD'), {'invoice': invoice, 'lineno': 2}),
 
    ])
 
    related = accrual.AccrualPostings(data.Posting.from_txn(txn))
 
    errors = list(related.report_inconsistencies())
 
    for post, err in itertools.zip_longest(txn.postings, errors):
 
        assert err.message == f'inconsistent cost for invoice {invoice}: {post.cost}'
 
        assert err.entry is txn
 
        assert err.source.get('lineno') == post.meta['lineno']
 

	
 
def test_make_consistent_not_needed():
 
    invoice = 'Invoices/ConsistentDoc.pdf'
 
    main_meta = {
 
        'entity': 'ConsistentTest',
 
        'invoice': 'Invoices/ConsistentDoc.pdf',
 
    }
 
    other_meta = {key: f'{key}.pdf' for key in CONSISTENT_METADATA}
 
    # We intentionally make inconsistencies in "minor" metadata that shouldn't
 
    # split out the group.
 
    txn = testutil.Transaction(postings=[
 
        (ACCOUNTS[0], 20, {**other_meta, 'invoice': invoice}),
 
        (ACCOUNTS[0], 25, {'invoice': invoice}),
 
        (ACCOUNTS[0], 20, {**main_meta, **other_meta}),
 
        (ACCOUNTS[0], 25, {**main_meta}),
 
    ])
 
    related = accrual.AccrualPostings(data.Posting.from_txn(txn))
 
    consistent = related.make_consistent()
 
    actual_key, actual_postings = next(consistent)
 
    assert actual_key == invoice
 
    assert actual_key == main_meta['invoice']
 
    assert actual_postings is related
 
    assert next(consistent, None) is None
 

	
 
@pytest.mark.parametrize('acct_name,invoice,day', testutil.combine_values(
 
    ACCOUNTS,
 
    ['FIXME', '', None, *testutil.NON_STRING_METADATA_VALUES],
 
    itertools.count(1),
 
))
 
def test_make_consistent_bad_invoice(acct_name, invoice, day):
 
    txn = testutil.Transaction(date=datetime.date(2019, 1, day), postings=[
 
        (acct_name, index * 10, {'invoice': invoice})
 
        (acct_name, index * 10, {'invoice': invoice, 'entity': f'BadInvoice{day}'})
 
        for index in range(1, 4)
 
    ])
 
    related = accrual.AccrualPostings(data.Posting.from_txn(txn))
 
    consistent = dict(related.make_consistent())
 
    assert len(consistent) == 1
 
    actual = consistent.get(f'{acct_name} None {invoice}')
 
    key = next(iter(consistent))
 
    assert acct_name in key
 
    if invoice:
 
        assert str(invoice) in key
 
    actual = consistent[key]
 
    assert actual
 
    assert len(actual) == 3
 
    for act_post, exp_post in zip(actual, txn.postings):
 
        assert act_post.units == exp_post.units
 
        assert act_post.meta.get('invoice') == invoice
 

	
 
def test_make_consistent_across_accounts():
 
    invoice = 'Invoices/CrossAccount.pdf'
 
    txn = testutil.Transaction(date=datetime.date(2019, 2, 1), postings=[
 
        (acct_name, 100, {'invoice': invoice})
 
        (acct_name, 100, {'invoice': invoice, 'entity': 'CrossAccount'})
 
        for acct_name in ACCOUNTS
 
    ])
 
    related = accrual.AccrualPostings(data.Posting.from_txn(txn))
 
    consistent = dict(related.make_consistent())
 
    assert len(consistent) == len(ACCOUNTS)
 
    for acct_name in ACCOUNTS:
 
        actual = consistent[f'{invoice} {acct_name}']
 
        assert len(actual) == 1
 
        assert actual[0].account == acct_name
 
    for key, posts in consistent.items():
 
        assert len(posts) == 1
 
        assert posts.account in key
 

	
 
def test_make_consistent_both_invoice_and_account():
 
    txn = testutil.Transaction(date=datetime.date(2019, 2, 2), postings=[
 
        (acct_name, 150) for acct_name in ACCOUNTS
 
    ])
 
    related = accrual.AccrualPostings(data.Posting.from_txn(txn))
 
    consistent = dict(related.make_consistent())
 
    assert len(consistent) == len(ACCOUNTS)
 
    for acct_name in ACCOUNTS:
 
        actual = consistent[f'{acct_name} None None']
 
        assert len(actual) == 1
 
        assert actual[0].account == acct_name
 
    for key, posts in consistent.items():
 
        assert len(posts) == 1
 
        assert posts.account in key
 

	
 
@pytest.mark.parametrize('acct_name', ACCOUNTS)
 
def test_make_consistent_across_entity(acct_name):
 
    amt_sign = operator.pos if acct_name.startswith('Assets') else operator.neg
 
    txn = testutil.Transaction(postings=[
 
        (acct_name, amt_sign(n), {'invoice': 'Inv/1.pdf', 'entity': f'Entity{n}'})
 
        for n in range(1, 4)
 
    ])
 
    related = accrual.AccrualPostings(data.Posting.from_txn(txn))
 
    consistent = dict(related.make_consistent())
 
    assert len(consistent) == 3
 
    for key, posts in consistent.items():
 
        assert len(posts) == 1
 
        assert len(posts.accrued_entities) == 1
 
        assert next(posts.entities()) in key
 

	
 
@pytest.mark.parametrize('acct_name', ACCOUNTS)
 
def test_make_consistent_entity_differs_accrual_payment(acct_name):
 
    invoice = 'Invoices/DifferPay.pdf'
 
    txn = testutil.Transaction(postings=[
 
        # Depending on the account, the order of the accrual and payment might
 
        # be swapped here, but that shouldn't matter.
 
        (acct_name, 125, {'invoice': invoice, 'entity': 'Positive'}),
 
        (acct_name, -125, {'invoice': invoice, 'entity': 'Negative'}),
 
    ])
 
    related = accrual.AccrualPostings(data.Posting.from_txn(txn))
 
    consistent = related.make_consistent()
 
    _, actual = next(consistent)
 
    assert actual is related
 
    assert next(consistent, None) is None
 

	
 
def check_output(output, expect_patterns):
 
    output.seek(0)
 
    testutil.check_lines_match(iter(output), expect_patterns)
 

	
 
def run_outgoing(invoice, postings, rt_client=None):
 
    if rt_client is None:
 
        rt_client = RTClient()
 
    if not isinstance(postings, core.RelatedPostings):
 
        postings = accruals_by_meta(postings, invoice, wrap_type=accrual.AccrualPostings)
 
    output = io.StringIO()
 
    report = accrual.OutgoingReport(rt_client, output)
...
 
@@ -650,24 +686,35 @@ def test_aging_report(accrual_postings):
 
    # Both these dates are chosen for their off-by-one potential:
 
    # the first is exactly 30 days after the 2010-06-10 payable;
 
    # the second is exactly 60 days after the 2010-05-15 receivable.
 
    (datetime.date(2010, 7, 10), 1, 4),
 
    (datetime.date(2010, 7, 14), 2, 4),
 
])
 
def test_aging_report_date_cutoffs(accrual_postings, date, recv_end, pay_end):
 
    expect_recv = AGING_AR[:recv_end]
 
    expect_pay = AGING_AP[:pay_end]
 
    output = run_aging_report(accrual_postings, date)
 
    check_aging_ods(output, date, expect_recv, expect_pay)
 

	
 
def test_aging_report_entity_consistency(accrual_postings):
 
    output = run_aging_report((
 
        post for post in accrual_postings
 
        if post.meta.get('rt-id') == 'rt:480'
 
        and post.units.number < 0
 
    ))
 
    check_aging_ods(output, None, [], [
 
        AgingRow.make_simple('2010-04-15', 'MultiPartyA', 125, 'rt:480/4800'),
 
        AgingRow.make_simple('2010-04-15', 'MultiPartyB', 125, 'rt:480/4800'),
 
    ])
 

	
 
def run_main(arglist, config=None):
 
    if config is None:
 
        config = testutil.TestConfig(
 
            books_path=testutil.test_path('books/accruals.beancount'),
 
            rt_client=RTClient(),
 
        )
 
    output = io.StringIO()
 
    errors = io.StringIO()
 
    retcode = accrual.main(arglist, output, errors, config)
 
    return retcode, output, errors
 

	
 
def check_main_fails(arglist, config, error_flags, error_patterns):
0 comments (0 inline, 0 general)