Changeset - d3e0a380736d
[Not reviewed]
0 2 0
Brett Smith - 4 years ago 2020-05-28 20:42:36
brettcsmith@brettcsmith.org
accrual: Introduce logging infrastructure.
2 files changed with 43 insertions and 34 deletions:
0 comments (0 inline, 0 general)
conservancy_beancount/reports/accrual.py
Show inline comments
...
 
@@ -49,32 +49,33 @@ option::
 
# it under the terms of the GNU Affero General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU Affero General Public License for more details.
 
#
 
# You should have received a copy of the GNU Affero General Public License
 
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 

	
 
import argparse
 
import collections
 
import datetime
 
import enum
 
import logging
 
import re
 
import sys
 

	
 
from typing import (
 
    Callable,
 
    Dict,
 
    Iterable,
 
    Iterator,
 
    Mapping,
 
    NamedTuple,
 
    Optional,
 
    Sequence,
 
    Set,
 
    TextIO,
 
    Tuple,
 
)
...
 
@@ -85,66 +86,68 @@ from ..beancount_types import (
 
    Transaction,
 
)
 

	
 
import rt
 

	
 
from beancount.parser import printer as bc_printer
 

	
 
from . import core
 
from .. import config as configmod
 
from .. import data
 
from .. import filters
 
from .. import rtutil
 

	
 
PostGroups = Mapping[Optional[MetaValue], core.RelatedPostings]
 
RTObject = Mapping[str, str]
 

	
 
_logger = logging.getLogger('conservancy_beancount.reports.accrual')
 

	
 
class Account(NamedTuple):
 
    name: str
 
    balance_paid: Callable[[core.Balance], bool]
 

	
 

	
 
class AccrualAccount(enum.Enum):
 
    PAYABLE = Account('Liabilities:Payable', core.Balance.ge_zero)
 
    RECEIVABLE = Account('Assets:Receivable', core.Balance.le_zero)
 

	
 
    @classmethod
 
    def account_names(cls) -> Iterator[str]:
 
        return (acct.value.name for acct in cls)
 

	
 
    @classmethod
 
    def classify(cls, related: core.RelatedPostings) -> 'AccrualAccount':
 
        for account in cls:
 
            account_name = account.value.name
 
            if all(post.account.is_under(account_name) for post in related):
 
                return account
 
        raise ValueError("unrecognized account set in related postings")
 

	
 
    @classmethod
 
    def filter_paid_accruals(cls, groups: PostGroups) -> PostGroups:
 
        return {
 
            key: related
 
            for key, related in groups.items()
 
            if not cls.classify(related).value.balance_paid(related.balance())
 
        }
 

	
 

	
 
class BaseReport:
 
    def __init__(self, out_file: TextIO, err_file: TextIO) -> None:
 
    def __init__(self, out_file: TextIO) -> None:
 
        self.out_file = out_file
 
        self.err_file = err_file
 
        self.logger = _logger.getChild(type(self).__name__)
 

	
 
    def _since_last_nonzero(self, posts: core.RelatedPostings) -> core.RelatedPostings:
 
        retval = core.RelatedPostings()
 
        for post in posts:
 
            if retval.balance().is_zero():
 
                retval.clear()
 
            retval.add(post)
 
        return retval
 

	
 
    def _report(self,
 
                invoice: str,
 
                posts: core.RelatedPostings,
 
                index: int,
 
    ) -> Iterable[str]:
 
        raise NotImplementedError("BaseReport._report")
 

	
...
 
@@ -157,68 +160,67 @@ class BaseReport:
 
class BalanceReport(BaseReport):
 
    def _report(self,
 
                invoice: str,
 
                posts: core.RelatedPostings,
 
                index: int,
 
    ) -> Iterable[str]:
 
        posts = self._since_last_nonzero(posts)
 
        balance = posts.balance()
 
        date_s = posts[0].meta.date.strftime('%Y-%m-%d')
 
        if index:
 
            yield ""
 
        yield f"{invoice}:"
 
        yield f"  {balance} outstanding since {date_s}"
 

	
 

	
 
class OutgoingReport(BaseReport):
 
    def __init__(self, rt_client: rt.Rt, out_file: TextIO, err_file: TextIO) -> None:
 
    def __init__(self, rt_client: rt.Rt, out_file: TextIO) -> None:
 
        super().__init__(out_file)
 
        self.rt_client = rt_client
 
        self.rt_wrapper = rtutil.RT(rt_client)
 
        self.out_file = out_file
 
        self.err_file = err_file
 

	
 
    def _primary_rt_id(self, posts: core.RelatedPostings) -> rtutil.TicketAttachmentIds:
 
        rt_ids = posts.all_meta_links('rt-id')
 
        rt_ids_count = len(rt_ids)
 
        if rt_ids_count != 1:
 
            raise ValueError(f"{rt_ids_count} rt-id links found")
 
        parsed = rtutil.RT.parse(rt_ids.pop())
 
        if parsed is None:
 
            raise ValueError("rt-id is not a valid RT reference")
 
        else:
 
            return parsed
 

	
 
    def _report(self,
 
                invoice: str,
 
                posts: core.RelatedPostings,
 
                index: int,
 
    ) -> Iterable[str]:
 
        posts = self._since_last_nonzero(posts)
 
        try:
 
            ticket_id, _ = self._primary_rt_id(posts)
 
            ticket = self.rt_client.get_ticket(ticket_id)
 
            # Note we only use this when ticket is None.
 
            errmsg = f"ticket {ticket_id} not found"
 
        except (ValueError, rt.RtError) as error:
 
            ticket = None
 
            errmsg = error.args[0]
 
        if ticket is None:
 
            print("error: can't generate outgoings report for {}"
 
                  " because no RT ticket available: {}".format(
 
                      invoice, errmsg,
 
                  ), file=self.err_file)
 
            self.logger.error(
 
                "can't generate outgoings report for %s because no RT ticket available: %s",
 
                invoice, errmsg,
 
            )
 
            return
 

	
 
        try:
 
            rt_requestor = self.rt_client.get_user(ticket['Requestors'][0])
 
        except (IndexError, rt.RtError):
 
            rt_requestor = None
 
        if rt_requestor is None:
 
            requestor = ''
 
            requestor_name = ''
 
        else:
 
            requestor_name = (
 
                rt_requestor.get('RealName')
 
                or ticket.get('CF.{payment-to}')
 
                or ''
 
            )
 
            requestor = f'{requestor_name} <{rt_requestor["EmailAddress"]}>'.strip()
...
 
@@ -366,71 +368,80 @@ You can either specify a fiscal year, or a negative offset from the current
 
fiscal year, to start loading entries from. The default is -1 (start from the
 
previous fiscal year).
 
""")
 
    parser.add_argument(
 
        'search',
 
        nargs=argparse.ZERO_OR_MORE,
 
        help="""Report on accruals that match this criteria. The format is
 
NAME=TERM. TERM is a link or word that must exist in a posting's NAME
 
metadata to match. A single ticket number is a shortcut for
 
`rt-id=rt:NUMBER`. Any other link, including an RT attachment link in
 
`TIK/ATT` format, is a shortcut for `invoice=LINK`.
 
""")
 
    args = parser.parse_args(arglist)
 
    args.search_terms = [SearchTerm.parse(s) for s in args.search]
 
    return args
 

	
 
def setup_logger(logger: logging.Logger, loglevel: int, stream: TextIO=sys.stderr) -> None:
 
    formatter = logging.Formatter('%(name)s: %(levelname)s: %(message)s')
 
    handler = logging.StreamHandler(stream)
 
    handler.setFormatter(formatter)
 
    logger.addHandler(handler)
 
    logger.setLevel(loglevel)
 

	
 
def main(arglist: Optional[Sequence[str]]=None,
 
         stdout: TextIO=sys.stdout,
 
         stderr: TextIO=sys.stderr,
 
         config: Optional[configmod.Config]=None,
 
         logger: Optional[logging.Logger]=None,
 
) -> int:
 
    if logger is None:
 
        global _logger
 
        _logger = logger = logging.getLogger('accrual-report')
 
        setup_logger(_logger, logging.INFO, stderr)
 
    returncode = 0
 
    args = parse_arguments(arglist)
 
    if config is None:
 
        config = configmod.Config()
 
        config.load_file()
 
    books_loader = config.books_loader()
 
    if books_loader is not None:
 
        entries, load_errors, _ = books_loader.load_fy_range(args.since)
 
    else:
 
        entries = []
 
        source = {
 
            'filename': str(config.config_file_path()),
 
            'lineno': 1,
 
        }
 
        load_errors = [Error(source, "no books to load in configuration", None)]
 
    postings = filter_search(data.Posting.from_entries(entries), args.search_terms)
 
    groups = core.RelatedPostings.group_by_meta(postings, 'invoice')
 
    groups = AccrualAccount.filter_paid_accruals(groups) or groups
 
    meta_errors = consistency_check(groups)
 
    for error in load_errors:
 
        bc_printer.print_error(error, file=stderr)
 
        returncode |= ReturnFlag.LOAD_ERRORS
 
    for error in meta_errors:
 
        bc_printer.print_error(error, file=stderr)
 
        returncode |= ReturnFlag.CONSISTENCY_ERRORS
 
    if args.report_type is None:
 
        args.report_type = ReportType.default_for(groups)
 
    if not groups:
 
        print("warning: no matching entries found to report", file=stderr)
 
        logger.warning("no matching entries found to report")
 
        returncode |= ReturnFlag.NOTHING_TO_REPORT
 
    report: Optional[BaseReport] = None
 
    if args.report_type is ReportType.OUTGOING:
 
        rt_client = config.rt_client()
 
        if rt_client is None:
 
            print(
 
                "error: unable to generate outgoing report: RT client is required",
 
                file=stderr,
 
            )
 
            logger.error("unable to generate outgoing report: RT client is required")
 
        else:
 
            report = OutgoingReport(rt_client, stdout, stderr)
 
            report = OutgoingReport(rt_client, stdout)
 
    else:
 
        report = args.report_type.value(stdout, stderr)
 
        report = args.report_type.value(stdout)
 
    if report is None:
 
        returncode |= ReturnFlag.REPORT_ERRORS
 
    else:
 
        report.run(groups)
 
    return 0 if returncode == 0 else 16 + returncode
 

	
 
if __name__ == '__main__':
 
    exit(main())
tests/test_reports_accrual.py
Show inline comments
...
 
@@ -246,93 +246,91 @@ def test_consistency_check_when_inconsistent(meta_key, account):
 
            f'inconsistent {meta_key} for invoice {invoice}: debit',
 
    ]), 1):
 
        assert actual.message == exp_msg
 
        assert actual.entry is txn
 
        assert actual.source.get('lineno') == exp_lineno
 

	
 
def check_output(output, expect_patterns):
 
    output.seek(0)
 
    testutil.check_lines_match(iter(output), expect_patterns)
 

	
 
def run_outgoing(invoice, postings, rt_client=None):
 
    if rt_client is None:
 
        rt_client = RTClient()
 
    if not isinstance(postings, core.RelatedPostings):
 
        postings = relate_accruals_by_meta(postings, invoice)
 
    output = io.StringIO()
 
    errors = io.StringIO()
 
    report = accrual.OutgoingReport(rt_client, output, errors)
 
    report = accrual.OutgoingReport(rt_client, output)
 
    report.run({invoice: postings})
 
    return output, errors
 
    return output
 

	
 
@pytest.mark.parametrize('invoice,expected', [
 
    ('rt:505/5050', "Zero balance outstanding since 2020-05-05"),
 
    ('rt:510/5100', "Zero balance outstanding since 2020-05-10"),
 
    ('rt:510/6100', "-280.00 USD outstanding since 2020-06-10"),
 
    ('rt://ticket/515/attachments/5150', "1,500.00 USD outstanding since 2020-05-15",),
 
])
 
def test_balance_report(accrual_postings, invoice, expected):
 
def test_balance_report(accrual_postings, invoice, expected, caplog):
 
    related = relate_accruals_by_meta(accrual_postings, invoice)
 
    output = io.StringIO()
 
    errors = io.StringIO()
 
    report = accrual.BalanceReport(output, errors)
 
    report = accrual.BalanceReport(output)
 
    report.run({invoice: related})
 
    assert not errors.getvalue()
 
    assert not caplog.records
 
    check_output(output, [invoice, expected])
 

	
 
def test_outgoing_report(accrual_postings):
 
def test_outgoing_report(accrual_postings, caplog):
 
    invoice = 'rt:510/6100'
 
    output, errors = run_outgoing(invoice, accrual_postings)
 
    output = run_outgoing(invoice, accrual_postings)
 
    rt_url = RTClient.DEFAULT_URL[:-9]
 
    rt_id_url = rf'\b{re.escape(f"{rt_url}Ticket/Display.html?id=510")}\b'
 
    contract_url = rf'\b{re.escape(f"{rt_url}Ticket/Attachment/4000/4000/contract.pdf")}\b'
 
    print(output.getvalue())
 
    assert not caplog.records
 
    check_output(output, [
 
        r'^PAYMENT FOR APPROVAL:$',
 
        r'^REQUESTOR: Mx\. 510 <mx510@example\.org>$',
 
        r'^TOTAL TO PAY: \$280\.00$',
 
        fr'^AGREEMENT: {contract_url}',
 
        r'^PAYMENT TO: Hon\. Mx\. 510$',
 
        r'^PAYMENT METHOD: payment method 510$',
 
        r'^BEANCOUNT ENTRIES:$',
 
        # For each transaction, check for the date line, a metadata, and the
 
        # Expenses posting.
 
        r'^\s*2020-06-10\s',
 
        fr'^\s+rt-id: "{rt_id_url}"$',
 
        r'^\s+Expenses:Services:Legal\s+220\.00 USD$',
 
        r'^\s*2020-06-12\s',
 
        fr'^\s+contract: "{contract_url}"$',
 
        r'^\s+Expenses:FilingFees\s+60\.00 USD$',
 
    ])
 

	
 
def test_outgoing_report_custom_field_fallbacks(accrual_postings):
 
def test_outgoing_report_custom_field_fallbacks(accrual_postings, caplog):
 
    invoice = 'rt:510/6100'
 
    rt_client = RTClient(want_cfs=False)
 
    output, errors = run_outgoing(invoice, accrual_postings, rt_client)
 
    assert not errors.getvalue()
 
    output = run_outgoing(invoice, accrual_postings, rt_client)
 
    assert not caplog.records
 
    check_output(output, [
 
        r'^PAYMENT FOR APPROVAL:$',
 
        r'^REQUESTOR: <mx510@example\.org>$',
 
        r'^PAYMENT TO:\s*$',
 
        r'^PAYMENT METHOD:\s*$',
 
    ])
 

	
 
def test_outgoing_report_fx_amounts(accrual_postings):
 
def test_outgoing_report_fx_amounts(accrual_postings, caplog):
 
    invoice = 'rt:520/5200'
 
    output, errors = run_outgoing(invoice, accrual_postings)
 
    assert not errors.getvalue()
 
    output = run_outgoing(invoice, accrual_postings)
 
    assert not caplog.records
 
    check_output(output, [
 
        r'^PAYMENT FOR APPROVAL:$',
 
        r'^REQUESTOR: Mx\. 520 <mx520@example\.org>$',
 
        r'^TOTAL TO PAY: 1,000\.00 EUR \(\$1,100.00\)$',
 
    ])
 

	
 
def run_main(arglist, config=None):
 
    if config is None:
 
        config = testutil.TestConfig(
 
            books_path=testutil.test_path('books/accruals.beancount'),
 
            rt_client=RTClient(),
 
        )
 
    output = io.StringIO()
 
    errors = io.StringIO()
 
    retcode = accrual.main(arglist, output, errors, config)
 
    return retcode, output, errors
...
 
@@ -404,26 +402,26 @@ def test_main_balance_report(arglist):
 
        r'\brt://ticket/515/attachments/5150:$',
 
        r'^\s+1,500\.00 USD outstanding since 2020-05-15$',
 
    ])
 

	
 
def test_main_no_books():
 
    check_main_fails([], testutil.TestConfig(), 1 | 8, [
 
        r':1: +no books to load in configuration\b',
 
    ])
 

	
 
@pytest.mark.parametrize('arglist', [
 
    ['499'],
 
    ['505/99999'],
 
    ['entity=NonExistent'],
 
])
 
def test_main_no_matches(arglist):
 
    check_main_fails(arglist, None, 8, [
 
        r'^warning: no matching entries found to report$',
 
        r': WARNING: no matching entries found to report$',
 
    ])
 

	
 
def test_main_no_rt():
 
    config = testutil.TestConfig(
 
        books_path=testutil.test_path('books/accruals.beancount'),
 
    )
 
    check_main_fails(['-t', 'out'], config, 4, [
 
        r'^error: unable to generate outgoing report: RT client is required\b',
 
        r': ERROR: unable to generate outgoing report: RT client is required\b',
 
    ])
0 comments (0 inline, 0 general)