Changeset - 175ac3bd7a30
[Not reviewed]
0 4 0
Brett Smith - 4 years ago 2020-06-11 19:27:36
brettcsmith@brettcsmith.org
accrual: Outgoing report groups by rt-id. RT#11594.
4 files changed with 78 insertions and 56 deletions:
0 comments (0 inline, 0 general)
conservancy_beancount/reports/accrual.py
Show inline comments
...
 
@@ -498,29 +498,30 @@ class BalanceReport(BaseReport):
 
            yield ""
 
        yield f"{posts.invoice}:"
 
        yield f"  {posts.balance_at_cost()} outstanding since {date_s}"
 

	
 

	
 
class OutgoingReport(BaseReport):
 
    def __init__(self, rt_client: rt.Rt, out_file: TextIO) -> None:
 
        super().__init__(out_file)
 
        self.rt_client = rt_client
 
        self.rt_wrapper = rtutil.RT(rt_client)
 

	
 
    def _primary_rt_id(self, posts: AccrualPostings) -> rtutil.TicketAttachmentIds:
 
        rt_ids = list(posts.first_meta_links('rt-id'))
 
        rt_ids_count = len(rt_ids)
 
        if rt_ids_count != 1:
 
            raise ValueError(f"{rt_ids_count} rt-id links found")
 
        parsed = rtutil.RT.parse(rt_ids.pop())
 
        rt_id = posts.rt_id
 
        if rt_id is None:
 
            raise ValueError("no rt-id links found")
 
        elif isinstance(rt_id, Sentinel):
 
            raise ValueError("multiple rt-id links found")
 
        parsed = rtutil.RT.parse(rt_id)
 
        if parsed is None:
 
            raise ValueError("rt-id is not a valid RT reference")
 
        else:
 
            return parsed
 

	
 
    def _report(self, posts: AccrualPostings, index: int) -> Iterable[str]:
 
        posts = posts.since_last_nonzero()
 
        try:
 
            ticket_id, _ = self._primary_rt_id(posts)
 
            ticket = self.rt_client.get_ticket(ticket_id)
 
            # Note we only use this when ticket is None.
 
            errmsg = f"ticket {ticket_id} not found"
...
 
@@ -589,38 +590,28 @@ class ReportType(enum.Enum):
 
    AGE = AGING
 
    BAL = BALANCE
 
    OUT = OUTGOING
 
    OUTGOINGS = OUTGOING
 

	
 
    @classmethod
 
    def by_name(cls, name: str) -> 'ReportType':
 
        try:
 
            return cls[name.upper()]
 
        except KeyError:
 
            raise ValueError(f"unknown report type {name!r}") from None
 

	
 
    @classmethod
 
    def default_for(cls, groups: PostGroups) -> 'ReportType':
 
        if len(groups) == 1 and all(
 
                group.accrual_type is AccrualAccount.PAYABLE
 
                and not group.is_paid()
 
                for group in groups.values()
 
        ):
 
            return cls.OUTGOING
 
        else:
 
            return cls.BALANCE
 

	
 

	
 
class ReturnFlag(enum.IntFlag):
 
    LOAD_ERRORS = 1
 
    # 2 was used in the past, it can probably be reclaimed.
 
    REPORT_ERRORS = 4
 
    NOTHING_TO_REPORT = 8
 

	
 

	
 
def filter_search(postings: Iterable[data.Posting],
 
                  search_terms: Iterable[cliutil.SearchTerm],
 
) -> Iterable[data.Posting]:
 
    accounts = tuple(AccrualAccount.account_names())
 
    postings = (post for post in postings if post.account.is_under(*accounts))
 
    for query in search_terms:
 
        postings = query.filter_postings(postings)
 
    return postings
...
 
@@ -678,75 +669,84 @@ def main(arglist: Optional[Sequence[str]]=None,
 
         config: Optional[configmod.Config]=None,
 
) -> int:
 
    if cliutil.is_main_script(PROGNAME):
 
        global logger
 
        logger = logging.getLogger(PROGNAME)
 
        sys.excepthook = cliutil.ExceptHook(logger)
 
    args = parse_arguments(arglist)
 
    cliutil.setup_logger(logger, args.loglevel, stderr)
 
    if config is None:
 
        config = configmod.Config()
 
        config.load_file()
 

	
 
    returncode = 0
 
    books_loader = config.books_loader()
 
    if books_loader is None:
 
        entries, load_errors, _ = books.Loader.load_none(config.config_file_path())
 
    elif args.report_type is ReportType.AGING:
 
        entries, load_errors, _ = books_loader.load_all()
 
    else:
 
        entries, load_errors, _ = books_loader.load_all(args.since)
 
    filters.remove_opening_balance_txn(entries)
 

	
 
    returncode = 0
 
    postings = filter_search(data.Posting.from_entries(entries), args.search_terms)
 
    groups: PostGroups = dict(AccrualPostings.group_by_first_meta_link(postings, 'invoice'))
 
    for error in load_errors:
 
        bc_printer.print_error(error, file=stderr)
 
        returncode |= ReturnFlag.LOAD_ERRORS
 
    if not groups:
 

	
 
    postings = list(filter_search(
 
        data.Posting.from_entries(entries), args.search_terms,
 
    ))
 
    if not postings:
 
        logger.warning("no matching entries found to report")
 
        returncode |= ReturnFlag.NOTHING_TO_REPORT
 

	
 
    groups = {
 
        key: posts
 
        for source_posts in groups.values()
 
        for key, posts in source_posts.make_consistent()
 
    }
 
    groups: PostGroups
 
    if args.report_type is None or args.report_type is ReportType.OUTGOING:
 
        groups = dict(AccrualPostings.group_by_first_meta_link(postings, 'rt-id'))
 
        if (args.report_type is None
 
            and len(groups) == 1
 
            and all(g.accrual_type is AccrualAccount.PAYABLE and not g.is_paid()
 
                    for g in groups.values())
 
        ):
 
            args.report_type = ReportType.OUTGOING
 
    if args.report_type is not ReportType.OUTGOING:
 
        groups = {
 
            key: group
 
            for _, source in AccrualPostings.group_by_first_meta_link(postings, 'invoice')
 
            for key, group in source.make_consistent()
 
        }
 
    if args.report_type is not ReportType.AGING:
 
        groups = {
 
            key: posts for key, posts in groups.items() if not posts.is_paid()
 
        } or groups
 
    del postings
 

	
 
    if args.report_type is None:
 
        args.report_type = ReportType.default_for(groups)
 
    report: Optional[BaseReport] = None
 
    output_path: Optional[Path] = None
 
    if args.report_type is ReportType.AGING:
 
        rt_client = config.rt_client()
 
        if rt_client is None:
 
            logger.error("unable to generate aging report: RT client is required")
 
        else:
 
            now = datetime.datetime.now()
 
            if args.output_file is None:
 
                args.output_file = Path(now.strftime('AgingReport_%Y-%m-%d_%H:%M.ods'))
 
                logger.info("Writing report to %s", args.output_file)
 
            out_bin = cliutil.bytes_output(args.output_file, stdout)
 
            report = AgingReport(rt_client, out_bin)
 
    elif args.report_type is ReportType.OUTGOING:
 
        rt_client = config.rt_client()
 
        if rt_client is None:
 
            logger.error("unable to generate outgoing report: RT client is required")
 
        else:
 
            out_file = cliutil.text_output(args.output_file, stdout)
 
            report = OutgoingReport(rt_client, out_file)
 
    else:
 
        out_file = cliutil.text_output(args.output_file, stdout)
 
        report = args.report_type.value(out_file)
 
        report = BalanceReport(out_file)
 

	
 
    if report is None:
 
        returncode |= ReturnFlag.REPORT_ERRORS
 
    else:
 
        report.run(groups)
 
    return 0 if returncode == 0 else 16 + returncode
 

	
 
if __name__ == '__main__':
 
    exit(main())
setup.py
Show inline comments
 
#!/usr/bin/env python3
 

	
 
from setuptools import setup
 

	
 
setup(
 
    name='conservancy_beancount',
 
    description="Plugin, library, and reports for reading Conservancy's books",
 
    version='1.1.8',
 
    version='1.1.9',
 
    author='Software Freedom Conservancy',
 
    author_email='info@sfconservancy.org',
 
    license='GNU AGPLv3+',
 

	
 
    install_requires=[
 
        'babel>=2.6',  # Debian:python3-babel
 
        'beancount>=2.2',  # Debian:beancount
 
        # 1.4.1 crashes when trying to save some documents.
 
        'odfpy>=1.4.0,!=1.4.1',  # Debian:python3-odf
 
        'PyYAML>=3.0',  # Debian:python3-yaml
 
        'regex',  # Debian:python3-regex
 
        'rt>=2.0',
tests/books/accruals.beancount
Show inline comments
...
 
@@ -62,32 +62,40 @@
 
  invoice: "rt:480/4800"
 
  project: "Conservancy"
 
  Liabilities:Payable:Accounts  125 USD
 
  Assets:Checking  -125 USD
 

	
 
2010-04-20 * "MultiPartyB" "Payment for 480"
 
  rt-id: "rt:480"
 
  invoice: "rt:480/4800"
 
  project: "Conservancy"
 
  Liabilities:Payable:Accounts  125 USD
 
  Assets:Checking  -125 USD
 

	
 
2010-04-30 ! "Vendor" "Travel reimbursement"
 
2010-04-25 ! "Vendor" "First trip travel reimbursement"
 
  rt-id: "rt:310"
 
  contract: "rt:310/3100"
 
  invoice: "FIXME"  ; still waiting on them to send it
 
  project: "Conservancy"
 
  Liabilities:Payable:Accounts  -200 USD
 
  Expenses:Travel  200 USD
 

	
 
2010-04-30 * "Vendor" "Second trip travel reimbursement"
 
  rt-id: "rt:310"
 
  contract: "rt:310/3100"
 
  invoice: "rt:310/3120"
 
  project: "Conservancy"
 
  Liabilities:Payable:Accounts  -220 USD
 
  Expenses:Travel  220 USD
 

	
 
2010-05-05 * "DonorA" "Donation pledge"
 
  rt-id: "rt:505"
 
  invoice: "rt:505/5050"
 
  approval: "rt:505/5040"
 
  project: "Conservancy"
 
  Income:Donations  -2,500 EUR {1.100 USD}
 
  Assets:Receivable:Accounts  2,500 EUR {1.100 USD}
 

	
 
2010-05-10 * "Lawyer" "April legal services"
 
  rt-id: "rt:510"
 
  invoice: "rt:510/5100"
 
  contract: "rt:510/4000"
tests/test_reports_accrual.py
Show inline comments
...
 
@@ -97,46 +97,51 @@ class AgingRow(NamedTuple):
 
        assert usd_cell.value_type == 'currency'
 
        assert usd_cell.value == self.at_cost.number
 
        assert next(cells).text == '\0'.join(self.project)
 
        for index, cell in enumerate(cells):
 
            links = cell.getElementsByType(odf.text.A)
 
            assert len(links) == len(cell.childNodes)
 
        assert index >= 1
 

	
 

	
 
AGING_AP = [
 
    AgingRow.make_simple('2010-03-06', 'EarlyBird', -125, 'rt:44/440'),
 
    AgingRow.make_simple('2010-03-30', 'EarlyBird', 75, 'rt:490/4900'),
 
    AgingRow.make_simple('2010-04-30', 'Vendor', 200, 'FIXME'),
 
    AgingRow.make_simple('2010-04-25', 'Vendor', 200, 'FIXME'),
 
    AgingRow.make_simple('2010-04-30', 'Vendor', 220, 'rt:310/3120'),
 
    AgingRow.make_simple('2010-06-10', 'Lawyer', 280, 'rt:510/6100'),
 
    AgingRow.make_simple('2010-06-18', 'EuroGov', 1100, 'rt:520/5200',
 
                         orig_amount=[testutil.Amount(1000, 'EUR')]),
 
]
 

	
 
AGING_AR = [
 
    AgingRow.make_simple('2010-03-05', 'EarlyBird', -500, 'rt:40/400'),
 
    AgingRow.make_simple('2010-05-15', 'MatchingProgram', 1500,
 
                         'rt://ticket/515/attachments/5150'),
 
    AgingRow.make_simple('2010-06-15', 'GrantCo', 11500, 'rt:470/4700',
 
                         project='Development Grant'),
 
]
 

	
 
class RTClient(testutil.RTClient):
 
    TICKET_DATA = {
 
        '40': [
 
            ('400', 'invoice feb.csv', 'text/csv', '40.4k'),
 
        ],
 
        '44': [
 
            ('440', 'invoice feb.csv', 'text/csv', '40.4k'),
 
        ],
 
        '310': [
 
            ('3100', 'VendorContract.pdf', 'application/pdf', '1.7m'),
 
            ('3120', 'VendorInvoiceB.pdf', 'application/pdf', '1.8m'),
 
        ],
 
        '490': [],
 
        '505': [],
 
        '510': [
 
            ('4000', 'contract.pdf', 'application/pdf', '1.4m'),
 
            ('5100', 'invoice april.pdf', 'application/pdf', '1.5m'),
 
            ('5105', 'payment.png', 'image/png', '51.5k'),
 
            ('6100', 'invoice may.pdf', 'application/pdf', '1.6m'),
 
        ],
 
        '515': [],
 
        '520': [],
 
    }
 

	
...
 
@@ -408,98 +413,107 @@ def test_make_consistent_entity_differs_accrual_payment(acct_name):
 
        (acct_name, -125, {'invoice': invoice, 'entity': 'Negative'}),
 
    ])
 
    related = accrual.AccrualPostings(data.Posting.from_txn(txn))
 
    consistent = related.make_consistent()
 
    _, actual = next(consistent)
 
    assert actual is related
 
    assert next(consistent, None) is None
 

	
 
def check_output(output, expect_patterns):
 
    output.seek(0)
 
    testutil.check_lines_match(iter(output), expect_patterns)
 

	
 
def run_outgoing(invoice, postings, rt_client=None):
 
def run_outgoing(rt_id, postings, rt_client=None):
 
    if rt_client is None:
 
        rt_client = RTClient()
 
    if not isinstance(postings, core.RelatedPostings):
 
        postings = accruals_by_meta(postings, invoice, wrap_type=accrual.AccrualPostings)
 
        postings = accruals_by_meta(postings, rt_id, 'rt-id', wrap_type=accrual.AccrualPostings)
 
    output = io.StringIO()
 
    report = accrual.OutgoingReport(rt_client, output)
 
    report.run({invoice: postings})
 
    report.run({rt_id: postings})
 
    return output
 

	
 
@pytest.mark.parametrize('invoice,expected', [
 
    ('rt:505/5050', "Zero balance outstanding since 2010-05-05"),
 
    ('rt:510/5100', "Zero balance outstanding since 2010-05-10"),
 
    ('rt:510/6100', "-280.00 USD outstanding since 2010-06-10"),
 
    ('rt://ticket/515/attachments/5150', "1,500.00 USD outstanding since 2010-05-15",),
 
])
 
def test_balance_report(accrual_postings, invoice, expected, caplog):
 
    related = accruals_by_meta(accrual_postings, invoice, wrap_type=accrual.AccrualPostings)
 
    output = io.StringIO()
 
    report = accrual.BalanceReport(output)
 
    report.run({invoice: related})
 
    assert not caplog.records
 
    check_output(output, [invoice, expected])
 

	
 
def test_outgoing_report(accrual_postings, caplog):
 
    invoice = 'rt:510/6100'
 
    output = run_outgoing(invoice, accrual_postings)
 
    output = run_outgoing('rt:510', accrual_postings)
 
    rt_url = RTClient.DEFAULT_URL[:-9]
 
    rt_id_url = rf'\b{re.escape(f"{rt_url}Ticket/Display.html?id=510")}\b'
 
    contract_url = rf'\b{re.escape(f"{rt_url}Ticket/Attachment/4000/4000/contract.pdf")}\b'
 
    assert not caplog.records
 
    check_output(output, [
 
        r'^PAYMENT FOR APPROVAL:$',
 
        r'^REQUESTOR: Mx\. 510 <mx510@example\.org>$',
 
        r'^TOTAL TO PAY: \$280\.00$',
 
        fr'^AGREEMENT: {contract_url}',
 
        r'^PAYMENT TO: Hon\. Mx\. 510$',
 
        r'^PAYMENT METHOD: payment method 510$',
 
        r'^BEANCOUNT ENTRIES:$',
 
        # For each transaction, check for the date line, a metadata, and the
 
        # Expenses posting.
 
        r'^\s*2010-06-10\s',
 
        fr'^\s+rt-id: "{rt_id_url}"$',
 
        r'^\s+Expenses:Services:Legal\s+220\.00 USD$',
 
        r'^\s*2010-06-12\s',
 
        fr'^\s+contract: "{contract_url}"$',
 
        r'^\s+Expenses:FilingFees\s+60\.00 USD$',
 
    ])
 

	
 
def test_outgoing_report_custom_field_fallbacks(accrual_postings, caplog):
 
    invoice = 'rt:510/6100'
 
    rt_client = RTClient(want_cfs=False)
 
    output = run_outgoing(invoice, accrual_postings, rt_client)
 
    output = run_outgoing('rt:510', accrual_postings, rt_client)
 
    assert not caplog.records
 
    check_output(output, [
 
        r'^PAYMENT FOR APPROVAL:$',
 
        r'^REQUESTOR: <mx510@example\.org>$',
 
        r'^PAYMENT TO:\s*$',
 
        r'^PAYMENT METHOD:\s*$',
 
    ])
 

	
 
def test_outgoing_report_fx_amounts(accrual_postings, caplog):
 
    invoice = 'rt:520/5200'
 
    output = run_outgoing(invoice, accrual_postings)
 
    output = run_outgoing('rt:520 rt:525', accrual_postings)
 
    assert not caplog.records
 
    check_output(output, [
 
        r'^PAYMENT FOR APPROVAL:$',
 
        r'^REQUESTOR: Mx\. 520 <mx520@example\.org>$',
 
        r'^TOTAL TO PAY: 1,000\.00 EUR \(\$1,100.00\)$',
 
    ])
 

	
 
def test_outgoing_report_multi_invoice(accrual_postings, caplog):
 
    output = run_outgoing('rt:310', accrual_postings)
 
    assert not caplog.records
 
    check_output(output, [
 
        r'^PAYMENT FOR APPROVAL:$',
 
        r'^REQUESTOR: Mx\. 310 <mx310@example\.org>$',
 
        r'^TOTAL TO PAY: \$420.00$',
 
    ])
 

	
 
def test_outgoing_report_without_rt_id(accrual_postings, caplog):
 
    invoice = 'rt://ticket/515/attachments/5150'
 
    output = run_outgoing(invoice, accrual_postings)
 
    related = accruals_by_meta(
 
        accrual_postings, invoice, wrap_type=accrual.AccrualPostings,
 
    )
 
    output = run_outgoing(None, related)
 
    assert caplog.records
 
    log = caplog.records[0]
 
    assert log.message.startswith(
 
        f"can't generate outgoings report for {invoice} because no RT ticket available:",
 
    )
 
    assert not output.getvalue()
 

	
 
def run_aging_report(postings, today=None):
 
    if today is None:
 
        today = datetime.date.today()
 
    postings = (
 
        post for post in postings
...
 
@@ -515,26 +529,26 @@ def run_aging_report(postings, today=None):
 
    report = accrual.AgingReport(rt_client, output, today)
 
    report.run(groups)
 
    return output
 

	
 
def test_aging_report(accrual_postings):
 
    output = run_aging_report(accrual_postings)
 
    check_aging_ods(output)
 

	
 
@pytest.mark.parametrize('date,recv_end,pay_end', [
 
    # Both these dates are chosen for their off-by-one potential:
 
    # the first is exactly 30 days after the 2010-06-10 payable;
 
    # the second is exactly 60 days after the 2010-05-15 receivable.
 
    (datetime.date(2010, 7, 10), 1, 4),
 
    (datetime.date(2010, 7, 14), 2, 4),
 
    (datetime.date(2010, 7, 10), 1, 5),
 
    (datetime.date(2010, 7, 14), 2, 5),
 
])
 
def test_aging_report_date_cutoffs(accrual_postings, date, recv_end, pay_end):
 
    expect_recv = AGING_AR[:recv_end]
 
    expect_pay = AGING_AP[:pay_end]
 
    if 10 <= date.day < 12:
 
        # Take the 60 USD posting out of the invoice 510/6100 payable.
 
        expect_pay[-1] = expect_pay[-1]._replace(
 
            at_cost=testutil.Amount(expect_pay[-1].at_cost.number - 60),
 
        )
 
    output = run_aging_report(accrual_postings, date)
 
    check_aging_ods(output, date, expect_recv, expect_pay)
 

	
...
 
@@ -597,41 +611,41 @@ def test_output_excludes_payments(arglist):
 
    (['40'], 'rt:40/400'),
 
    (['44/440'], 'rt:44/440'),
 
])
 
def test_output_payments_when_only_match(arglist, expect_invoice):
 
    retcode, output, errors = run_main(arglist)
 
    assert not errors.getvalue()
 
    assert retcode == 0
 
    check_output(output, [
 
        rf'^{re.escape(expect_invoice)}:$',
 
        r' outstanding since ',
 
    ])
 

	
 
@pytest.mark.parametrize('arglist', [
 
    ['510'],
 
    ['510/6100'],
 
    ['entity=Lawyer'],
 
@pytest.mark.parametrize('arglist,expect_amount', [
 
    (['310'], 420),
 
    (['310/3120'], 220),
 
    (['entity=Vendor'], 420),
 
])
 
def test_main_outgoing_report(arglist):
 
def test_main_outgoing_report(arglist, expect_amount):
 
    retcode, output, errors = run_main(arglist)
 
    assert not errors.getvalue()
 
    assert retcode == 0
 
    rt_url = RTClient.DEFAULT_URL[:-9]
 
    rt_id_url = re.escape(f'<{rt_url}Ticket/Display.html?id=510>')
 
    contract_url = re.escape(f'<{rt_url}Ticket/Attachment/4000/4000/contract.pdf>')
 
    rt_id_url = re.escape(f'<{rt_url}Ticket/Display.html?id=310>')
 
    contract_url = re.escape(f'<{rt_url}Ticket/Attachment/3120/3120/VendorContract.pdf>')
 
    check_output(output, [
 
        r'^REQUESTOR: Mx\. 510 <mx510@example\.org>$',
 
        r'^TOTAL TO PAY: \$280\.00$',
 
        r'^\s*2010-06-12\s',
 
        r'^\s+Expenses:FilingFees\s+60\.00 USD$',
 
        r'^REQUESTOR: Mx\. 310 <mx310@example\.org>$',
 
        rf'^TOTAL TO PAY: \${expect_amount}\.00$',
 
        r'^\s*2010-04-30\s',
 
        r'^\s+Expenses:Travel\s+220 USD$',
 
    ])
 

	
 
@pytest.mark.parametrize('arglist', [
 
    ['-t', 'balance'],
 
    ['515/5150'],
 
    ['entity=MatchingProgram'],
 
])
 
def test_main_balance_report(arglist):
 
    retcode, output, errors = run_main(arglist)
 
    assert not errors.getvalue()
 
    assert retcode == 0
 
    check_output(output, [
0 comments (0 inline, 0 general)