Changeset - 5085d4d8ef79
[Not reviewed]
0 5 0
Brett Smith - 4 years ago 2020-06-23 18:47:03
brettcsmith@brettcsmith.org
accrual: Outgoing report sets RT CFs for outgoing payment.
5 files changed with 106 insertions and 11 deletions:
0 comments (0 inline, 0 general)
conservancy_beancount/reports/accrual.py
Show inline comments
...
 
@@ -62,24 +62,25 @@ type you want with the ``--report-type`` option::
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU Affero General Public License for more details.
 
#
 
# You should have received a copy of the GNU Affero General Public License
 
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 

	
 
import argparse
 
import collections
 
import datetime
 
import enum
 
import logging
 
import re
 
import sys
 

	
 
from pathlib import Path
 

	
 
from typing import (
 
    cast,
 
    Any,
 
    BinaryIO,
 
    Callable,
 
    Iterable,
 
    Iterator,
 
    List,
...
 
@@ -462,24 +463,39 @@ class AgingReport(BaseReport):
 
class BalanceReport(BaseReport):
 
    def _report(self, posts: AccrualPostings, index: int) -> Iterable[str]:
 
        posts = posts.since_last_nonzero()
 
        date_s = posts[0].meta.date.strftime('%Y-%m-%d')
 
        balance_s = posts.balance_at_cost().format(zero="Zero balance")
 
        if index:
 
            yield ""
 
        yield f"{posts.invoice}:"
 
        yield f"  {balance_s} outstanding since {date_s}"
 

	
 

	
 
class OutgoingReport(BaseReport):
 
    PAYMENT_METHODS = {
 
        'ach': 'ACH',
 
        'check': 'Check',
 
        'creditcard': 'Credit Card',
 
        'debitcard': 'Debit Card',
 
        'fxwire': 'International Wire',
 
        'paypal': 'PayPal',
 
        'uswire': 'Domestic Wire',
 
        'vendorportal': 'Vendor Portal',
 
    }
 
    PAYMENT_METHOD_RE = re.compile(
 
        rf'^([A-Z]{{3}})\s+({"|".join(PAYMENT_METHODS)})$',
 
        re.IGNORECASE,
 
    )
 

	
 
    def __init__(self, rt_wrapper: rtutil.RT, out_file: TextIO) -> None:
 
        super().__init__(out_file)
 
        self.rt_wrapper = rt_wrapper
 
        self.rt_client = rt_wrapper.rt
 

	
 
    def _primary_rt_id(self, posts: AccrualPostings) -> rtutil.TicketAttachmentIds:
 
        rt_id = posts.rt_id
 
        if rt_id is None:
 
            raise ValueError("no rt-id links found")
 
        elif isinstance(rt_id, Sentinel):
 
            raise ValueError("multiple rt-id links found")
 
        parsed = rtutil.RT.parse(rt_id)
...
 
@@ -513,54 +529,104 @@ class OutgoingReport(BaseReport):
 
            requestor = ''
 
            requestor_name = ''
 
        else:
 
            requestor_name = (
 
                rt_requestor.get('RealName')
 
                or ticket.get('CF.{payment-to}')
 
                or ''
 
            )
 
            requestor = f'{requestor_name} <{rt_requestor["EmailAddress"]}>'.strip()
 

	
 
        balance_s = posts.end_balance.format(None)
 
        raw_balance = -posts.balance()
 
        payment_amount = raw_balance.format('¤¤ #,##0.00')
 
        if raw_balance != posts.end_balance:
 
            payment_amount += f' ({balance_s})'
 
            balance_s = f'{raw_balance} ({balance_s})'
 

	
 
        payment_to = ticket.get('CF.{payment-to}') or requestor_name
 
        contract_links = list(posts.all_meta_links('contract'))
 
        if contract_links:
 
            contract_s = ' , '.join(self.rt_wrapper.iter_urls(
 
                contract_links, missing_fmt='<BROKEN RT LINK: {}>',
 
            ))
 
        else:
 
            contract_s = "NO CONTRACT GOVERNS THIS TRANSACTION"
 
        projects = [v for v in posts.meta_values('project')
 
                    if isinstance(v, str)]
 

	
 
        yield "PAYMENT FOR APPROVAL:"
 
        yield f"REQUESTOR: {requestor}"
 
        yield f"PAYMENT TO: {payment_to}"
 
        yield f"TOTAL TO PAY: {balance_s}"
 
        yield f"AGREEMENT: {contract_s}"
 
        yield f"PAYMENT TO: {ticket.get('CF.{payment-to}') or requestor_name}"
 
        yield f"PAYMENT METHOD: {ticket.get('CF.{payment-method}', '')}"
 
        yield f"PROJECT: {', '.join(projects)}"
 
        yield "\nBEANCOUNT ENTRIES:\n"
 

	
 
        last_txn: Optional[Transaction] = None
 
        for post in posts:
 
            txn = post.meta.txn
 
            if txn is not last_txn:
 
                last_txn = txn
 
                txn = self.rt_wrapper.txn_with_urls(txn, '{}')
 
                # Suppress payment-method metadata from the report.
 
                txn.meta.pop('payment-method', None)
 
                for txn_post in txn.postings:
 
                    if txn_post.meta:
 
                        txn_post.meta.pop('payment-method', None)
 
                yield bc_printer.format_entry(txn)
 

	
 
        cf_targets = {
 
            'payment-amount': payment_amount,
 
            'payment-to': payment_to,
 
        }
 
        payment_methods = posts.meta_values('payment-method')
 
        payment_methods.discard(None)
 
        payment_method_count = len(payment_methods)
 
        if payment_method_count != 1:
 
            self.logger.warning(
 
                "cannot set payment-method for rt:%s: %s metadata values found",
 
                ticket_id, payment_method_count,
 
            )
 
        else:
 
            payment_method = payment_methods.pop()
 
            if isinstance(payment_method, str):
 
                match = self.PAYMENT_METHOD_RE.fullmatch(payment_method)
 
            else:
 
                match = None
 
            if match is None:
 
                self.logger.warning(
 
                    "cannot set payment-method for rt:%s: invalid value %r",
 
                    ticket_id, payment_method,
 
                )
 
            else:
 
                cf_targets['payment-method'] = '{} {}'.format(
 
                    match.group(1).upper(),
 
                    self.PAYMENT_METHODS[match.group(2).lower()],
 
                )
 

	
 
        cf_updates = {
 
            f'CF_{key}': value
 
            for key, value in cf_targets.items()
 
            if ticket.get(f'CF.{{{key}}}') != value
 
        }
 
        if cf_updates:
 
            try:
 
                ok = self.rt_client.edit_ticket(ticket_id, **cf_updates)
 
            except rt.RtError:
 
                self.logger.debug("RT exception on edit_ticket", exc_info=True)
 
                ok = False
 
            if not ok:
 
                self.logger.warning("failed to set custom fields for rt:%s", ticket_id)
 

	
 

	
 
class ReportType(enum.Enum):
 
    AGING = AgingReport
 
    BALANCE = BalanceReport
 
    OUTGOING = OutgoingReport
 
    AGE = AGING
 
    BAL = BALANCE
 
    OUT = OUTGOING
 
    OUTGOINGS = OUTGOING
 

	
 
    @classmethod
 
    def by_name(cls, name: str) -> 'ReportType':
setup.py
Show inline comments
 
#!/usr/bin/env python3
 

	
 
from setuptools import setup
 

	
 
setup(
 
    name='conservancy_beancount',
 
    description="Plugin, library, and reports for reading Conservancy's books",
 
    version='1.3.1',
 
    version='1.4.0',
 
    author='Software Freedom Conservancy',
 
    author_email='info@sfconservancy.org',
 
    license='GNU AGPLv3+',
 

	
 
    install_requires=[
 
        'babel>=2.6',  # Debian:python3-babel
 
        'beancount>=2.2',  # Debian:beancount
 
        # 1.4.1 crashes when trying to save some documents.
 
        'odfpy>=1.4.0,!=1.4.1',  # Debian:python3-odf
 
        'PyYAML>=3.0',  # Debian:python3-yaml
 
        'regex',  # Debian:python3-regex
 
        'rt>=2.0',
tests/books/accruals.beancount
Show inline comments
...
 
@@ -67,50 +67,53 @@
 
2010-04-20 * "MultiPartyB" "Payment for 480"
 
  rt-id: "rt:480"
 
  invoice: "rt:480/4800"
 
  project: "Conservancy"
 
  Liabilities:Payable:Accounts  125 USD
 
  Assets:Checking  -125 USD
 

	
 
2010-04-25 ! "Vendor" "First trip travel reimbursement"
 
  rt-id: "rt:310"
 
  contract: "rt:310/3100"
 
  invoice: "FIXME"  ; still waiting on them to send it
 
  project: "Conservancy"
 
  payment-method: "USD USWire"
 
  Liabilities:Payable:Accounts  -200 USD
 
  Expenses:Travel  200 USD
 

	
 
2010-04-30 * "Vendor" "Second trip travel reimbursement"
 
  rt-id: "rt:310"
 
  contract: "rt:310/3100"
 
  invoice: "rt:310/3120"
 
  project: "Conservancy"
 
  payment-method: "USD Check"
 
  Liabilities:Payable:Accounts  -220 USD
 
  Expenses:Travel  220 USD
 

	
 
2010-05-05 * "DonorA" "Donation pledge"
 
  rt-id: "rt:505"
 
  invoice: "rt:505/5050"
 
  approval: "rt:505/5040"
 
  project: "Conservancy"
 
  Income:Donations  -2,500 EUR {1.100 USD}
 
  Assets:Receivable:Accounts  2,500 EUR {1.100 USD}
 

	
 
2010-05-10 * "Lawyer" "April legal services"
 
  rt-id: "rt:510"
 
  invoice: "rt:510/5100"
 
  contract: "rt:510/4000"
 
  project: "Conservancy"
 
  Expenses:Services:Legal  200.00 USD
 
  Liabilities:Payable:Accounts  -200.00 USD
 
  payment-method: "USD ACH"
 

	
 
2010-05-15 * "MatchingProgram" "May matched donations"
 
  invoice: "rt://ticket/515/attachments/5150"
 
  approval: "rt://ticket/515/attachments/5140"
 
  project: "Conservancy"
 
  Income:Donations  -1500.00 USD
 
  Assets:Receivable:Accounts  1500.00 USD
 

	
 
2010-05-20 * "DonorA" "Donation made"
 
  rt-id: "rt:505"
 
  invoice: "rt:505/5050"
 
  project: "Conservancy"
...
 
@@ -125,48 +128,51 @@
 
  Liabilities:Payable:Accounts  200.00 USD
 
  contract: "rt:510/4000"
 
  Assets:Checking  -200.00 USD
 
  receipt: "rt:510/5105"
 

	
 
2010-06-10 * "Lawyer" "May legal services"
 
  rt-id: "rt:510"
 
  invoice: "rt:510/6100"
 
  contract: "rt:510/4000"
 
  project: "Conservancy"
 
  Expenses:Services:Legal  220.00 USD
 
  Liabilities:Payable:Accounts  -220.00 USD
 
  payment-method: "USD ACH"
 

	
 
2010-06-12 * "Lawyer" "Additional legal fees for May"
 
  rt-id: "rt:510"
 
  invoice: "rt:510/6100"
 
  contract: "rt:510/4000"
 
  project: "Conservancy"
 
  Expenses:FilingFees  60.00 USD
 
  Liabilities:Payable:Accounts  -60.00 USD
 
  payment-method: "USD ACH"
 

	
 
2010-06-15 * "GrantCo" "2010Q2 grant"
 
  rt-id: "rt:470"
 
  invoice: "rt:470/4700"
 
  project: "Development Grant"
 
  Assets:Receivable:Accounts  5500 USD
 
  Income:Donations           -5500 USD
 

	
 
2010-06-18 * "EuroGov" "European legal fees"
 
  ; Multiple rt-ids are used to test proper handling for both
 
  ; searching and generating the outgoing report.
 
  rt-id: "rt:520 rt:525"
 
  invoice: "rt:520/5200"
 
  contract: "rt:520/5220"
 
  project: "Conservancy"
 
  Liabilities:Payable:Accounts  -1,000 EUR {1.100 USD}
 
  payment-method: "eur fxwire"
 
  Expenses:FilingFees  1,000 EUR {1.100 USD}
 

	
 
2010-06-20 * "StateGov" "Business registration"
 
  ; Intentionally has no rt-id
 
  invoice: "Invoices/2010StateRegistration.pdf"
 
  project: "Conservancy"
 
  Liabilities:Payable:Accounts  -50 USD
 
  Expenses:FilingFees  50 USD
 

	
 
2010-09-15 * "GrantCo" "2010Q3 grant"
 
  rt-id: "rt:470"
 
  invoice: "rt:470/4700"
tests/test_reports_accrual.py
Show inline comments
...
 
@@ -438,75 +438,92 @@ def run_outgoing(rt_id, postings, rt_client=None):
 
    ('rt:510/6100', "-280.00 USD outstanding since 2010-06-10"),
 
    ('rt://ticket/515/attachments/5150', "1,500.00 USD outstanding since 2010-05-15",),
 
])
 
def test_balance_report(accrual_postings, invoice, expected, caplog):
 
    related = accruals_by_meta(accrual_postings, invoice, wrap_type=accrual.AccrualPostings)
 
    output = io.StringIO()
 
    report = accrual.BalanceReport(output)
 
    report.run({invoice: related})
 
    assert not caplog.records
 
    check_output(output, [invoice, expected])
 

	
 
def test_outgoing_report(accrual_postings, caplog):
 
    output = run_outgoing('rt:510', accrual_postings)
 
    rt_client = RTClient()
 
    output = run_outgoing('rt:510', accrual_postings, rt_client)
 
    rt_url = RTClient.DEFAULT_URL[:-9]
 
    rt_id_url = rf'\b{re.escape(f"{rt_url}Ticket/Display.html?id=510")}\b'
 
    contract_url = rf'\b{re.escape(f"{rt_url}Ticket/Attachment/4000/4000/contract.pdf")}\b'
 
    assert not caplog.records
 
    check_output(output, [
 
        r'^PAYMENT FOR APPROVAL:$',
 
        r'^REQUESTOR: Mx\. 510 <mx510@example\.org>$',
 
        r'^PAYMENT TO: Hon\. Mx\. 510$',
 
        r'^TOTAL TO PAY: \$280\.00$',
 
        fr'^AGREEMENT: {contract_url}',
 
        r'^PAYMENT TO: Hon\. Mx\. 510$',
 
        r'^PAYMENT METHOD: payment method 510$',
 
        r'^BEANCOUNT ENTRIES:$',
 
        # For each transaction, check for the date line, a metadata, and the
 
        # Expenses posting.
 
        r'^\s*2010-06-10\s',
 
        fr'^\s+rt-id: "{rt_id_url}"$',
 
        r'^\s+Expenses:Services:Legal\s+220\.00 USD$',
 
        r'^\s*2010-06-12\s',
 
        fr'^\s+contract: "{contract_url}"$',
 
        r'^\s+Expenses:FilingFees\s+60\.00 USD$',
 
    ])
 
    assert rt_client.edits == {'510': {
 
        'CF_payment-amount': 'USD 280.00',
 
        'CF_payment-method': 'USD ACH',
 
    }}
 
    assert 'payment-method:' not in output.getvalue()
 

	
 
def test_outgoing_report_custom_field_fallbacks(accrual_postings, caplog):
 
    rt_client = RTClient(want_cfs=False)
 
    output = run_outgoing('rt:510', accrual_postings, rt_client)
 
    assert not caplog.records
 
    check_output(output, [
 
        r'^PAYMENT FOR APPROVAL:$',
 
        r'^REQUESTOR: <mx510@example\.org>$',
 
        r'^PAYMENT TO:\s*$',
 
        r'^PAYMENT METHOD:\s*$',
 
    ])
 

	
 
def test_outgoing_report_fx_amounts(accrual_postings, caplog):
 
    output = run_outgoing('rt:520 rt:525', accrual_postings)
 
    rt_client = RTClient()
 
    output = run_outgoing('rt:520 rt:525', accrual_postings, rt_client)
 
    assert not caplog.records
 
    check_output(output, [
 
        r'^PAYMENT FOR APPROVAL:$',
 
        r'^REQUESTOR: Mx\. 520 <mx520@example\.org>$',
 
        r'^TOTAL TO PAY: 1,000\.00 EUR \(\$1,100.00\)$',
 
    ])
 
    assert rt_client.edits == {'520': {
 
        'CF_payment-amount': 'EUR 1,000.00 ($1,100.00)',
 
        'CF_payment-method': 'EUR International Wire',
 
    }}
 
    assert 'payment-method:' not in output.getvalue()
 

	
 
def test_outgoing_report_multi_invoice(accrual_postings, caplog):
 
    output = run_outgoing('rt:310', accrual_postings)
 
    assert not caplog.records
 
    rt_client = RTClient()
 
    output = run_outgoing('rt:310', accrual_postings, rt_client)
 
    log, = caplog.records
 
    assert log.levelname == 'WARNING'
 
    assert log.message.startswith('cannot set payment-method for rt:310: ')
 
    check_output(output, [
 
        r'^PAYMENT FOR APPROVAL:$',
 
        r'^REQUESTOR: Mx\. 310 <mx310@example\.org>$',
 
        r'^TOTAL TO PAY: \$420.00$',
 
    ])
 
    assert rt_client.edits == {'310': {
 
        'CF_payment-amount': 'USD 420.00',
 
    }}
 
    assert 'payment-method:' not in output.getvalue()
 

	
 
def test_outgoing_report_without_rt_id(accrual_postings, caplog):
 
    invoice = 'rt://ticket/515/attachments/5150'
 
    related = accruals_by_meta(
 
        accrual_postings, invoice, wrap_type=accrual.AccrualPostings,
 
    )
 
    output = run_outgoing(None, related)
 
    assert caplog.records
 
    log = caplog.records[0]
 
    assert log.message.startswith(
 
        f"can't generate outgoings report for {invoice} because no RT ticket available:",
 
    )
tests/testutil.py
Show inline comments
...
 
@@ -339,24 +339,25 @@ class RTClient:
 
        if http_auth is None:
 
            self.user = default_login
 
            self.password = default_password
 
            self.auth_method = 'login'
 
            self.login_result = skip_login or None
 
        else:
 
            self.user = http_auth.username
 
            self.password = http_auth.password
 
            self.auth_method = type(http_auth).__name__
 
            self.login_result = True
 
        self.last_login = None
 
        self.want_cfs = want_cfs
 
        self.edits = {}
 

	
 
    def login(self, login=None, password=None):
 
        if login is None and password is None:
 
            login = self.user
 
            password = self.password
 
        self.login_result = bool(login and password and not password.startswith('bad'))
 
        self.last_login = (login, password, self.login_result)
 
        return self.login_result
 

	
 
    def get_attachments(self, ticket_id):
 
        try:
 
            return list(self.TICKET_DATA[str(ticket_id)])
...
 
@@ -393,28 +394,33 @@ class RTClient:
 
        ticket_id_s = str(ticket_id)
 
        if ticket_id_s not in self.TICKET_DATA:
 
            return None
 
        retval = {
 
            'id': 'ticket/{}'.format(ticket_id_s),
 
            'numerical_id': ticket_id_s,
 
            'Requestors': [
 
                f'mx{ticket_id_s}@example.org',
 
                'requestor2@example.org',
 
            ],
 
        }
 
        if self.want_cfs:
 
            retval['CF.{payment-method}'] = f'payment method {ticket_id_s}'
 
            retval['CF.{payment-amount}'] = ''
 
            retval['CF.{payment-method}'] = ''
 
            retval['CF.{payment-to}'] = f'Hon. Mx. {ticket_id_s}'
 
        return retval
 

	
 
    def edit_ticket(self, ticket_id, **kwargs):
 
        self.edits.setdefault(str(ticket_id), {}).update(kwargs)
 
        return True
 

	
 
    def get_user(self, user_id):
 
        user_id_s = str(user_id)
 
        match = re.search(r'(\d+)@', user_id_s)
 
        if match is None:
 
            email = f'mx{user_id_s}@example.org'
 
            user_id_num = int(user_id_s)
 
        else:
 
            email = user_id_s
 
            user_id_num = int(match.group(1))
 
        retval = {
 
            'id': f'user/{user_id_num}',
 
            'EmailAddress': email,
0 comments (0 inline, 0 general)