*,

                 _can_own: bool=False,

    ) -> None:

        super().__init__(source, _can_own=_can_own)

        # The following type declarations tell mypy about values set in the for

        # loop that are important enough to be referenced directly elsewhere.

        self.account: Union[data.Account, Sentinel]

        self.invoice: Union[MetaValue, Sentinel]

        for name, get_func in self._FIELDS.items():

            values = frozenset(get_func(post) for post in self)

            setattr(self, f'{name}s', values)

            if len(values) == 1:

                one_value = next(iter(values))

            else:

                one_value = self.INCONSISTENT

            setattr(self, name, one_value)

        if self.account is self.INCONSISTENT:

            self.accrual_type: Optional[AccrualAccount] = None

            self.end_balance = self.balance_at_cost()

            self.accrued_entities = self._collect_entities()

            self.paid_entities = self.accrued_entities

        else:

            self.accrual_type = AccrualAccount.classify(self)

            accrual_acct: Account = self.accrual_type.value

            norm_func = accrual_acct.norm_func

            self.end_balance = norm_func(self.balance_at_cost())

            self.accrued_entities = self._collect_entities(

                lambda post: norm_func(post.units).number > 0,

            )

            self.paid_entities = self._collect_entities(

                lambda post: norm_func(post.units).number < 0,

            )

    def _collect_entities(self,

                          pred: Callable[[data.Posting], bool]=bool,

                          default: str='<empty>',

    ) -> FrozenSet[MetaValue]:

        return frozenset(

            post.meta.get('entity') or default

            for post in self if pred(post)

        )

    def entities(self) -> Iterator[MetaValue]:

        yield from self.accrued_entities

        yield from self.paid_entities.difference(self.accrued_entities)

    def make_consistent(self) -> Iterator[Tuple[MetaValue, 'AccrualPostings']]:

        account_ok = isinstance(self.account, str)

        # `'/' in self.invoice` is just our heuristic to ensure that the

        # invoice metadata is "unique enough," and not just a placeholder

        # value like "FIXME". It can be refined if needed.

        invoice_ok = isinstance(self.invoice, str) and '/' in self.invoice

            yield (self.invoice, self)

            return

        groups = collections.defaultdict(list)

        for post in self:

        type_self = type(self)

        for group_key, posts in groups.items():

            yield group_key, type_self(posts, _can_own=True)

    def report_inconsistencies(self) -> Iterable[Error]:

        for field_name, get_func in self._FIELDS.items():

            if getattr(self, field_name) is self.INCONSISTENT:

                for post in self:

                    errmsg = 'inconsistent {} for invoice {}: {}'.format(

                        field_name.replace('_', '-'),

                        self.invoice or "<none>",

                        get_func(post),

                    )

                    yield Error(post.meta, errmsg, post.meta.txn)

        costs = collections.defaultdict(set)

        for post in self:

            costs[post.units.currency].add(post.cost)

        for code, currency_costs in costs.items():

            if len(currency_costs) > 1:

                for post in self:

                    if post.units.currency == code:

                        errmsg = 'inconsistent cost for invoice {}: {}'.format(

                            self.invoice or "<none>", post.cost,

                        )

                        yield Error(post.meta, errmsg, post.meta.txn)

    def is_paid(self, default: Optional[bool]=None) -> Optional[bool]:

        if self.accrual_type is None:

            return default

        else:

            return self.end_balance.le_zero()

    def is_zero(self, default: Optional[bool]=None) -> Optional[bool]:

        if self.accrual_type is None:

            return default

        else:

            return self.end_balance.is_zero()

    def since_last_nonzero(self) -> 'AccrualPostings':

        for index, (post, balance) in enumerate(self.iter_with_balance()):

            if balance.is_zero():

                start_index = index

        try:

            empty = start_index == index

        except NameError:

            empty = True

        return self if empty else self[start_index + 1:]

class BaseReport:

    def __init__(self, out_file: TextIO) -> None:

        self.out_file = out_file

        self.logger = logger.getChild(type(self).__name__)

        raise NotImplementedError("BaseReport._report")

    def run(self, groups: PostGroups) -> None:

        for index, invoice in enumerate(groups):

                print(line, file=self.out_file)

class AgingODS(core.BaseODS[AccrualPostings, Optional[data.Account]]):

    COLUMNS = [

        'Date',

        'Entity',

        'Invoice Amount',

        'Booked Amount',

        'Ticket',

        'Invoice',

    ]

    COL_COUNT = len(COLUMNS)

    def __init__(self,

                 rt_client: rt.Rt,

                 date: datetime.date,

                 logger: logging.Logger,

    ) -> None:

        super().__init__()

        self.rt_client = rt_client

        self.rt_wrapper = rtutil.RT(self.rt_client)

        self.date = date

        self.logger = logger

    def init_styles(self) -> None:

        super().init_styles()

        self.style_widecol = self.replace_child(

            self.document.automaticstyles,

            odf.style.Style,

            name='WideCol',

        )

        self.style_widecol.setAttribute('family', 'table-column')

        self.style_widecol.addElement(odf.style.TableColumnProperties(

            columnwidth='1.25in',

        ))

    def section_key(self, row: AccrualPostings) -> Optional[data.Account]:

        if isinstance(row.account, str):

            return row.account

        else:

            return None

    def start_spreadsheet(self) -> None:

        for accrual_type in AccrualAccount:

            self.use_sheet(accrual_type.name.title())

            for index in range(self.COL_COUNT):

                stylename = self.style_widecol if index else ''

        if row.end_balance.ge_zero():

            for index, threshold in enumerate(self.age_thresholds):

                if age >= threshold:

                    self.age_balances[index] += row.end_balance

                    break

            else:

                return

        raw_balance = row.balance()

        if row.accrual_type is not None:

            raw_balance = row.accrual_type.value.norm_func(raw_balance)

        if raw_balance == row.end_balance:

            amount_cell = odf.table.TableCell()

        else:

            amount_cell = self.balance_cell(raw_balance)

        self.add_row(

            self.date_cell(row[0].meta.date),

            self.multiline_cell(row.entities()),

            amount_cell,

            self.balance_cell(row.end_balance),

            self.multilink_cell(self._link_seq(row, 'rt-id')),

            self.multilink_cell(self._link_seq(row, 'invoice')),

        )

class AgingReport(BaseReport):

    def __init__(self,

                 rt_client: rt.Rt,

                 out_file: BinaryIO,

                 date: Optional[datetime.date]=None,

    ) -> None:

        if date is None:

            date = datetime.date.today()

        self.out_bin = out_file

        self.logger = logger.getChild(type(self).__name__)

        self.ods = AgingODS(rt_client, date, self.logger)

    def run(self, groups: PostGroups) -> None:

        rows = list(group for group in groups.values() if not group.is_zero())

        rows.sort(key=lambda related: (

            related.account,

            related[0].meta.date,

            min(related.entities()) if related.accrued_entities else '',

        ))

        self.ods.write(rows)

        self.ods.save_file(self.out_bin)

class BalanceReport(BaseReport):

        posts = posts.since_last_nonzero()

        date_s = posts[0].meta.date.strftime('%Y-%m-%d')

        if index:

            yield ""

        yield f"  {posts.balance_at_cost()} outstanding since {date_s}"

class OutgoingReport(BaseReport):

    def __init__(self, rt_client: rt.Rt, out_file: TextIO) -> None:

        super().__init__(out_file)

        self.rt_client = rt_client

        self.rt_wrapper = rtutil.RT(rt_client)

    def _primary_rt_id(self, posts: AccrualPostings) -> rtutil.TicketAttachmentIds:

        rt_ids = posts.all_meta_links('rt-id')

        rt_ids_count = len(rt_ids)

        if rt_ids_count != 1:

            raise ValueError(f"{rt_ids_count} rt-id links found")

        parsed = rtutil.RT.parse(rt_ids.pop())

        if parsed is None:

            raise ValueError("rt-id is not a valid RT reference")

        else:

            return parsed

        posts = posts.since_last_nonzero()

        try:

            ticket_id, _ = self._primary_rt_id(posts)

            ticket = self.rt_client.get_ticket(ticket_id)

            # Note we only use this when ticket is None.

            errmsg = f"ticket {ticket_id} not found"

        except (ValueError, rt.RtError) as error:

            ticket = None

            errmsg = error.args[0]

        if ticket is None:

            self.logger.error(

                "can't generate outgoings report for %s because no RT ticket available: %s",

            )

            return

        try:

            rt_requestor = self.rt_client.get_user(ticket['Requestors'][0])

        except (IndexError, rt.RtError):

            rt_requestor = None

        if rt_requestor is None:

            requestor = ''

            requestor_name = ''

        else:

            requestor_name = (

                rt_requestor.get('RealName')

                or ticket.get('CF.{payment-to}')

                or ''

            )

            requestor = f'{requestor_name} <{rt_requestor["EmailAddress"]}>'.strip()

        balance_s = posts.end_balance.format(None)

        raw_balance = -posts.balance()

        if raw_balance != posts.end_balance:

            balance_s = f'{raw_balance} ({balance_s})'

        contract_links = posts.all_meta_links('contract')

        if contract_links:

            contract_s = ' , '.join(self.rt_wrapper.iter_urls(

                contract_links, missing_fmt='<BROKEN RT LINK: {}>',

            ))

        else:

            contract_s = "NO CONTRACT GOVERNS THIS TRANSACTION"

        projects = [v for v in posts.meta_values('project')

                    if isinstance(v, str)]

        yield "PAYMENT FOR APPROVAL:"

        yield f"REQUESTOR: {requestor}"

        yield f"TOTAL TO PAY: {balance_s}"

        yield f"AGREEMENT: {contract_s}"

        yield f"PAYMENT TO: {ticket.get('CF.{payment-to}') or requestor_name}"

        yield f"PAYMENT METHOD: {ticket.get('CF.{payment-method}', '')}"

        yield f"PROJECT: {', '.join(projects)}"

        yield "\nBEANCOUNT ENTRIES:\n"

        last_txn: Optional[Transaction] = None

        for post in posts:

            txn = post.meta.txn

            if txn is not last_txn:

                last_txn = txn

                txn = self.rt_wrapper.txn_with_urls(txn, '{}')

#!/usr/bin/env python3

from setuptools import setup

setup(

    name='conservancy_beancount',

    description="Plugin, library, and reports for reading Conservancy's books",

    author='Software Freedom Conservancy',

    author_email='info@sfconservancy.org',

    license='GNU AGPLv3+',

    install_requires=[

        'babel>=2.6',  # Debian:python3-babel

        'beancount>=2.2',  # Debian:beancount

        # 1.4.1 crashes when trying to save some documents.

        'odfpy>=1.4.0,!=1.4.1',  # Debian:python3-odf

        'PyYAML>=3.0',  # Debian:python3-yaml

        'regex',  # Debian:python3-regex

        'rt>=2.0',

    ],

    setup_requires=[

        'pytest-mypy',

        'pytest-runner',  # Debian:python3-pytest-runner

    ],

    tests_require=[

        'mypy>=0.770',  # Debian:python3-mypy

        'pytest',  # Debian:python3-pytest

    ],

    packages=[

        'conservancy_beancount',

        'conservancy_beancount.plugin',

        'conservancy_beancount.reports',

    ],

    entry_points={

        'console_scripts': [

            'accrual-report = conservancy_beancount.reports.accrual:main',

        ],

    },

)

"""test_reports_accrual - Unit tests for accrual report"""

# Copyright © 2020  Brett Smith

#

# This program is free software: you can redistribute it and/or modify

# it under the terms of the GNU Affero General Public License as published by

# the Free Software Foundation, either version 3 of the License, or

# (at your option) any later version.

#

# This program is distributed in the hope that it will be useful,

# but WITHOUT ANY WARRANTY; without even the implied warranty of

# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

# GNU Affero General Public License for more details.

#

# You should have received a copy of the GNU Affero General Public License

# along with this program.  If not, see <https://www.gnu.org/licenses/>.

import collections

import copy

import datetime

import io

import itertools

import logging

import re

import babel.numbers

import odf.opendocument

import odf.table

import odf.text

import pytest

from . import testutil

from decimal import Decimal

from typing import NamedTuple, Optional, Sequence

from beancount.core import data as bc_data

from beancount import loader as bc_loader

from conservancy_beancount import data

from conservancy_beancount import rtutil

from conservancy_beancount.reports import accrual

from conservancy_beancount.reports import core

_accruals_load = bc_loader.load_file(testutil.test_path('books/accruals.beancount'))

ACCRUAL_TXNS = [

    entry for entry in _accruals_load[0]

    if hasattr(entry, 'narration')

    and entry.narration != 'Opening balances'

]

ACCRUALS_COUNT = sum(

    1

    for txn in ACCRUAL_TXNS

    for post in txn.postings

    if post.account.startswith(('Assets:Receivable:', 'Liabilities:Payable:'))

)

ACCOUNTS = [

    'Assets:Receivable:Accounts',

    'Assets:Receivable:Loans',

    'Liabilities:Payable:Accounts',

    'Liabilities:Payable:Vacation',

]

CONSISTENT_METADATA = [

    'contract',

    'purchase-order',

]

class AgingRow(NamedTuple):

    date: datetime.date

@pytest.mark.parametrize('meta_key,account', testutil.combine_values(

    ['approval', 'entity', 'fx-rate', 'statement'],

    ACCOUNTS,

))

def test_consistency_check_ignored_metadata(meta_key, account):

    invoice = f'test-{meta_key}-invoice'

    txn = testutil.Transaction(postings=[

        (account, 100, {'invoice': invoice, meta_key: 'credit'}),

        (account, -100, {'invoice': invoice, meta_key: 'debit'}),

    ])

    related = accrual.AccrualPostings(data.Posting.from_txn(txn))

    assert not list(related.report_inconsistencies())

@pytest.mark.parametrize('meta_key,account', testutil.combine_values(

    CONSISTENT_METADATA,

    ACCOUNTS,

))

def test_consistency_check_when_inconsistent(meta_key, account):

    invoice = f'test-{meta_key}-invoice'

    txn = testutil.Transaction(postings=[

        (account, 100, {'invoice': invoice, meta_key: 'credit', 'lineno': 1}),

        (account, -100, {'invoice': invoice, meta_key: 'debit', 'lineno': 2}),

    ])

    related = accrual.AccrualPostings(data.Posting.from_txn(txn))

    errors = list(related.report_inconsistencies())

    for exp_lineno, (actual, exp_msg) in enumerate(itertools.zip_longest(errors, [

            f'inconsistent {meta_key} for invoice {invoice}: credit',

            f'inconsistent {meta_key} for invoice {invoice}: debit',

    ]), 1):

        assert actual.message == exp_msg

        assert actual.entry is txn

        assert actual.source.get('lineno') == exp_lineno

def test_consistency_check_cost():

    account = ACCOUNTS[0]

    invoice = 'test-cost-invoice'

    txn = testutil.Transaction(postings=[

        (account, 100, 'EUR', ('1.1251', 'USD'), {'invoice': invoice, 'lineno': 1}),

        (account, -100, 'EUR', ('1.125', 'USD'), {'invoice': invoice, 'lineno': 2}),

    ])

    related = accrual.AccrualPostings(data.Posting.from_txn(txn))

    errors = list(related.report_inconsistencies())

    for post, err in itertools.zip_longest(txn.postings, errors):

        assert err.message == f'inconsistent cost for invoice {invoice}: {post.cost}'

        assert err.entry is txn

        assert err.source.get('lineno') == post.meta['lineno']

def test_make_consistent_not_needed():

    other_meta = {key: f'{key}.pdf' for key in CONSISTENT_METADATA}

    # We intentionally make inconsistencies in "minor" metadata that shouldn't

    # split out the group.

    txn = testutil.Transaction(postings=[

    ])

    related = accrual.AccrualPostings(data.Posting.from_txn(txn))

    consistent = related.make_consistent()

    actual_key, actual_postings = next(consistent)

    assert actual_postings is related

    assert next(consistent, None) is None

@pytest.mark.parametrize('acct_name,invoice,day', testutil.combine_values(

    ACCOUNTS,

    ['FIXME', '', None, *testutil.NON_STRING_METADATA_VALUES],

    itertools.count(1),

))

def test_make_consistent_bad_invoice(acct_name, invoice, day):

    txn = testutil.Transaction(date=datetime.date(2019, 1, day), postings=[

        for index in range(1, 4)

    ])

    related = accrual.AccrualPostings(data.Posting.from_txn(txn))

    consistent = dict(related.make_consistent())

    assert len(consistent) == 1

    assert actual

    assert len(actual) == 3

    for act_post, exp_post in zip(actual, txn.postings):

        assert act_post.units == exp_post.units

        assert act_post.meta.get('invoice') == invoice

def test_make_consistent_across_accounts():

    invoice = 'Invoices/CrossAccount.pdf'

    txn = testutil.Transaction(date=datetime.date(2019, 2, 1), postings=[

        for acct_name in ACCOUNTS

    ])

    related = accrual.AccrualPostings(data.Posting.from_txn(txn))

    consistent = dict(related.make_consistent())

    assert len(consistent) == len(ACCOUNTS)

def test_make_consistent_both_invoice_and_account():

    txn = testutil.Transaction(date=datetime.date(2019, 2, 2), postings=[

        (acct_name, 150) for acct_name in ACCOUNTS

    ])

    related = accrual.AccrualPostings(data.Posting.from_txn(txn))

    consistent = dict(related.make_consistent())

    assert len(consistent) == len(ACCOUNTS)

def check_output(output, expect_patterns):

    output.seek(0)

    testutil.check_lines_match(iter(output), expect_patterns)

def run_outgoing(invoice, postings, rt_client=None):

    if rt_client is None:

        rt_client = RTClient()

    if not isinstance(postings, core.RelatedPostings):

        postings = accruals_by_meta(postings, invoice, wrap_type=accrual.AccrualPostings)

    output = io.StringIO()

    report = accrual.OutgoingReport(rt_client, output)

    report.run({invoice: postings})

    return output

@pytest.mark.parametrize('invoice,expected', [

    ('rt:505/5050', "Zero balance outstanding since 2010-05-05"),

    ('rt:510/5100', "Zero balance outstanding since 2010-05-10"),

    ('rt:510/6100', "-280.00 USD outstanding since 2010-06-10"),

    ('rt://ticket/515/attachments/5150', "1,500.00 USD outstanding since 2010-05-15",),

])

def test_balance_report(accrual_postings, invoice, expected, caplog):

    related = accruals_by_meta(accrual_postings, invoice, wrap_type=accrual.AccrualPostings)

    output = io.StringIO()

    report = accrual.BalanceReport(output)

    report.run({invoice: related})

    assert not caplog.records

    check_output(output, [invoice, expected])

def test_outgoing_report(accrual_postings, caplog):

    invoice = 'rt:510/6100'

    output = run_outgoing(invoice, accrual_postings)

    rt_url = RTClient.DEFAULT_URL[:-9]

    rt_id_url = rf'\b{re.escape(f"{rt_url}Ticket/Display.html?id=510")}\b'

    contract_url = rf'\b{re.escape(f"{rt_url}Ticket/Attachment/4000/4000/contract.pdf")}\b'

    assert not caplog.records

    check_output(output, [

        r'^PAYMENT FOR APPROVAL:$',

        r'^REQUESTOR: Mx\. 510 <mx510@example\.org>$',

        r'^TOTAL TO PAY: \$280\.00$',

        fr'^AGREEMENT: {contract_url}',

        r'^PAYMENT TO: Hon\. Mx\. 510$',

        r'^PAYMENT METHOD: payment method 510$',

        r'^BEANCOUNT ENTRIES:$',

        # For each transaction, check for the date line, a metadata, and the

        # Expenses posting.

        r'^\s*2010-06-10\s',

        fr'^\s+rt-id: "{rt_id_url}"$',

        r'^TOTAL TO PAY: 1,000\.00 EUR \(\$1,100.00\)$',

    ])

def test_outgoing_report_without_rt_id(accrual_postings, caplog):

    invoice = 'rt://ticket/515/attachments/5150'

    output = run_outgoing(invoice, accrual_postings)

    assert caplog.records

    log = caplog.records[0]

    assert log.message.startswith(

        f"can't generate outgoings report for {invoice} because no RT ticket available:",

    )

    assert not output.getvalue()

def run_aging_report(postings, today=None):

    if today is None:

        today = datetime.date.today()

    postings = (

        post for post in postings

        if post.account.is_under('Assets:Receivable', 'Liabilities:Payable')

    )

    groups = {

        key: group

        for _, related in accrual.AccrualPostings.group_by_meta(postings, 'invoice')

        for key, group in related.make_consistent()

    }

    output = io.BytesIO()

    rt_client = RTClient()

    report = accrual.AgingReport(rt_client, output, today)

    report.run(groups)

    return output

def test_aging_report(accrual_postings):

    output = run_aging_report(accrual_postings)

    check_aging_ods(output)

@pytest.mark.parametrize('date,recv_end,pay_end', [

    # Both these dates are chosen for their off-by-one potential:

    # the first is exactly 30 days after the 2010-06-10 payable;

    # the second is exactly 60 days after the 2010-05-15 receivable.

    (datetime.date(2010, 7, 10), 1, 4),

    (datetime.date(2010, 7, 14), 2, 4),

])

def test_aging_report_date_cutoffs(accrual_postings, date, recv_end, pay_end):

    expect_recv = AGING_AR[:recv_end]

    expect_pay = AGING_AP[:pay_end]

    output = run_aging_report(accrual_postings, date)

    check_aging_ods(output, date, expect_recv, expect_pay)

def run_main(arglist, config=None):

    if config is None:

        config = testutil.TestConfig(

            books_path=testutil.test_path('books/accruals.beancount'),

            rt_client=RTClient(),

        )

    output = io.StringIO()

    errors = io.StringIO()

    retcode = accrual.main(arglist, output, errors, config)

    return retcode, output, errors

def check_main_fails(arglist, config, error_flags, error_patterns):

    retcode, output, errors = run_main(arglist, config)

    assert retcode > 16

    assert (retcode - 16) & error_flags

    check_output(errors, error_patterns)

    assert not output.getvalue()

@pytest.mark.parametrize('arglist', [

    ['--report-type=balance', 'entity=EarlyBird'],

    ['--report-type=outgoing', 'entity=EarlyBird'],

])

def test_output_excludes_payments(arglist):

    retcode, output, errors = run_main(arglist)

    assert not errors.getvalue()

    assert retcode == 0

    output.seek(0)

    for line in output:

        assert not re.match(r'\brt:4\d\b', line)

@pytest.mark.parametrize('arglist,expect_invoice', [

    (['40'], 'rt:40/400'),

    (['44/440'], 'rt:44/440'),

])

def test_output_payments_when_only_match(arglist, expect_invoice):

    retcode, output, errors = run_main(arglist)

    assert not errors.getvalue()

    assert retcode == 0

    check_output(output, [

        rf'^{re.escape(expect_invoice)}:$',

        r' outstanding since ',

    ])

@pytest.mark.parametrize('arglist', [

    ['510'],

    ['510/6100'],

    ['entity=Lawyer'],

])