Changeset - 35804db617a1
[Not reviewed]
0 7 0
Brett Smith - 4 years ago 2020-08-31 18:19:00
brettcsmith@brettcsmith.org
reports: All reports support rewrite rules.

I realized that if ledger-report supported rewrite rules, then it would
include all the information necessary to reproduce the numbers on the
statement of functional expenses.

With that, it was easy enough to add support to the rest of the reports for
consistency's sake.
7 files changed with 51 insertions and 20 deletions:
0 comments (0 inline, 0 general)
conservancy_beancount/cliutil.py
Show inline comments
 
"""cliutil - Utilities for CLI tools"""
 
PKGNAME = 'conservancy_beancount'
 
LICENSE = """
 
Copyright © 2020  Brett Smith
 

	
 
This program is free software: you can redistribute it and/or modify
 
it under the terms of the GNU Affero General Public License as published by
 
the Free Software Foundation, either version 3 of the License, or
 
(at your option) any later version.
 

	
 
This program is distributed in the hope that it will be useful,
 
but WITHOUT ANY WARRANTY; without even the implied warranty of
 
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
GNU Affero General Public License for more details.
 

	
 
You should have received a copy of the GNU Affero General Public License
 
along with this program.  If not, see <https://www.gnu.org/licenses/>."""
 

	
 
import argparse
 
import datetime
 
import enum
 
import io
 
import logging
 
import operator
 
import os
 
import pkg_resources
 
import re
 
import signal
 
import sys
 
import traceback
 
import types
 

	
 
from pathlib import Path
 

	
 
import rt.exceptions as rt_error
 

	
 
from . import data
 
from . import filters
 
from . import rtutil
 

	
 
from typing import (
 
    cast,
 
    Any,
 
    BinaryIO,
 
    Callable,
 
    IO,
 
    Iterable,
 
    NamedTuple,
 
    NoReturn,
 
    Optional,
 
    Sequence,
 
    TextIO,
 
    Type,
 
    Union,
 
)
 
from .beancount_types import (
 
    MetaKey,
 
)
 

	
 
OutputFile = Union[int, IO]
 

	
 
STDSTREAM_PATH = Path('-')
 
VERSION = pkg_resources.require(PKGNAME)[0].version
 

	
 
class ExceptHook:
 
    def __init__(self, logger: Optional[logging.Logger]=None) -> None:
 
        if logger is None:
 
            logger = logging.getLogger()
 
        self.logger = logger
 

	
 
    def __call__(self,
 
                 exc_type: Type[BaseException],
 
                 exc_value: BaseException,
 
                 exc_tb: types.TracebackType,
 
    ) -> NoReturn:
 
        error_type = type(exc_value).__name__
 
        msg = ": ".join(str(arg) for arg in exc_value.args)
 
        if isinstance(exc_value, KeyboardInterrupt):
 
            signal.signal(signal.SIGINT, signal.SIG_DFL)
 
            os.kill(0, signal.SIGINT)
 
            signal.pause()
 
        elif isinstance(exc_value, (
 
                rt_error.AuthorizationError,
 
                rt_error.NotAllowed,
 
        )):
 
            exitcode = os.EX_NOPERM
 
            error_type = "RT access denied"
 
        elif isinstance(exc_value, rt_error.ConnectionError):
 
            exitcode = os.EX_TEMPFAIL
 
            error_type = "RT connection error"
 
        elif isinstance(exc_value, rt_error.RtError):
 
            exitcode = os.EX_UNAVAILABLE
 
            error_type = f"RT {error_type}"
 
        elif isinstance(exc_value, OSError):
 
            if exc_value.filename is None:
 
                exitcode = os.EX_OSERR
 
                error_type = "OS error"
 
                msg = exc_value.strerror
 
            else:
 
                # There are more specific exit codes for input problems vs.
 
                # output problems, but without knowing how the file was
 
                # intended to be used, we can't use them.
 
                exitcode = os.EX_IOERR
 
                error_type = "I/O error"
 
                msg = f"{exc_value.filename}: {exc_value.strerror}"
 
        else:
 
            exitcode = os.EX_SOFTWARE
 
            error_type = f"internal {error_type}"
 
        self.logger.critical("%s%s%s", error_type, ": " if msg else "", msg)
 
        self.logger.debug(
 
            ''.join(traceback.format_exception(exc_type, exc_value, exc_tb)),
 
        )
 
        raise SystemExit(exitcode)
 

	
 

	
 
class ExitCode(enum.IntEnum):
 
    # BSD exit codes commonly used
 
    NoConfiguration = os.EX_CONFIG
 
    NoConfig = NoConfiguration
 
    NoDataFiltered = os.EX_DATAERR
 
    NoDataLoaded = os.EX_NOINPUT
 
    RewriteRulesError = os.EX_DATAERR
 

	
 
    # Our own exit codes, working down from that range
 
    BeancountErrors = 63
 

	
 

	
 
class InfoAction(argparse.Action):
 
    def __call__(self,
 
                 parser: argparse.ArgumentParser,
 
                 namespace: argparse.Namespace,
 
                 values: Union[Sequence[Any], str, None]=None,
 
                 option_string: Optional[str]=None,
 
    ) -> NoReturn:
 
        if isinstance(self.const, str):
 
            info = self.const
 
            exitcode = 0
 
        else:
 
            info, exitcode = self.const
 
        print(info)
 
        raise SystemExit(exitcode)
 

	
 

	
 
class LogLevel(enum.IntEnum):
 
    DEBUG = logging.DEBUG
 
    INFO = logging.INFO
 
    WARNING = logging.WARNING
 
    ERROR = logging.ERROR
 
    CRITICAL = logging.CRITICAL
 
    WARN = WARNING
 
    ERR = ERROR
 
    CRIT = CRITICAL
 

	
 
    @classmethod
 
    def from_arg(cls, arg: str) -> int:
 
        try:
 
            return cls[arg.upper()].value
 
        except KeyError:
 
            raise ValueError(f"unknown loglevel {arg!r}") from None
 

	
 
    @classmethod
 
    def choices(cls) -> Iterable[str]:
 
        for level in sorted(cls, key=operator.attrgetter('value')):
 
            yield level.name.lower()
 

	
 

	
 
class SearchTerm(NamedTuple):
 
    """NamedTuple representing a user's metadata filter
 

	
 
    SearchTerm knows how to parse and store posting metadata filters provided
 
    by the user in `key=value` format. Reporting tools can use this to filter
 
    postings that match the user's criteria, to report on subsets of the books.
 

	
 
    Typical usage looks like::
 

	
 
      argument_parser.add_argument(
 
        'search_terms',
 
        type=SearchTerm.arg_parser(),
 
        …,
 
      )
 

	
 
      args = argument_parser.parse_args(…)
 
      for query in args.search_terms:
 
        postings = query.filter_postings(postings)
 
    """
 
    meta_key: MetaKey
 
    pattern: str
 

	
 
    @classmethod
 
    def arg_parser(cls,
 
                   default_key: Optional[str]=None,
 
                   ticket_default_key: Optional[str]=None,
 
    ) -> Callable[[str], 'SearchTerm']:
 
        """Build a SearchTerm parser
 

	
 
        This method returns a function that can parse strings in ``key=value``
 
        format and return a corresponding SearchTerm.
 

	
 
        If you specify a default key, then strings that just specify a ``value``
 
        will be parsed as if they said ``default_key=value``. Otherwise,
 
        parsing strings without a metadata key will raise a ValueError.
 

	
 
        If you specify a default key ticket links, then values in the format
 
        ``number``, ``rt:number``, or ``rt://ticket/number`` will be parsed as
 
        if they said ``ticket_default_key=value``.
 
        """
 
        if ticket_default_key is None:
 
            ticket_default_key = default_key
 
        def parse_search_term(arg: str) -> 'SearchTerm':
 
            key: Optional[str] = None
 
            if re.match(r'^[a-z][-\w]*=', arg):
 
                key, _, raw_link = arg.partition('=')
 
            else:
 
                raw_link = arg
 
            rt_ids = rtutil.RT.parse(raw_link)
 
            if rt_ids is None:
 
                rt_ids = rtutil.RT.parse('rt:' + raw_link)
 
            if rt_ids is None:
 
                if key is None:
 
                    key = default_key
 
                pattern = r'(?:^|\s){}(?:\s|$)'.format(re.escape(raw_link))
 
            else:
 
                ticket_id, attachment_id = rt_ids
 
                if key is None:
 
                    if attachment_id is None:
 
                        key = ticket_default_key
 
                    else:
 
                        key = default_key
 
                pattern = rtutil.RT.metadata_regexp(
 
                    ticket_id,
 
                    attachment_id,
 
                    first_link_only=key == 'rt-id' and attachment_id is None,
 
                )
 
            if key is None:
 
                raise ValueError(f"invalid search term {arg!r}: no metadata key")
 
            return cls(key, pattern)
 
        return parse_search_term
 

	
 
    def filter_postings(self, postings: Iterable[data.Posting]) -> Iterable[data.Posting]:
 
        return filters.filter_meta_match(
 
            postings, self.meta_key, re.compile(self.pattern),
 
        )
 

	
 

	
 
def add_loglevel_argument(parser: argparse.ArgumentParser,
 
                          default: LogLevel=LogLevel.INFO) -> argparse.Action:
 
    return parser.add_argument(
 
        '--loglevel',
 
        metavar='LEVEL',
 
        default=default.value,
 
        type=LogLevel.from_arg,
 
        help="Show logs at this level and above."
 
        f" Specify one of {', '.join(LogLevel.choices())}."
 
        f" Default {default.name.lower()}.",
 
    )
 

	
 
def add_rewrite_rules_argument(parser: argparse.ArgumentParser) -> argparse.Action:
 
    return parser.add_argument(
 
        '--rewrite-rules', '--rewrites', '-r',
 
        action='append',
 
        default=[],
 
        metavar='PATH',
 
        type=Path,
 
        help="""Use rewrite rules from the given YAML file. You can specify
 
this option multiple times to load multiple sets of rewrite rules in order.
 
""")
 

	
 
def add_version_argument(parser: argparse.ArgumentParser) -> argparse.Action:
 
    progname = parser.prog or sys.argv[0]
 
    return parser.add_argument(
 
        '--version', '--copyright', '--license',
 
        action=InfoAction,
 
        nargs=0,
 
        const=f"{progname} version {VERSION}\n{LICENSE}",
 
        help="Show program version and license information",
 
    )
 

	
 
def date_arg(arg: str) -> datetime.date:
 
    return datetime.datetime.strptime(arg, '%Y-%m-%d').date()
 

	
 
def diff_year(date: datetime.date, diff: int) -> datetime.date:
 
    new_year = date.year + diff
 
    try:
 
        return date.replace(year=new_year)
 
    except ValueError:
 
        # The original date is Feb 29, which doesn't exist in the new year.
 
        if diff < 0:
 
            return datetime.date(new_year, 2, 28)
 
        else:
 
            return datetime.date(new_year, 3, 1)
 

	
 
def year_or_date_arg(arg: str) -> Union[int, datetime.date]:
 
    """Get either a date or a year (int) from an argument string
 

	
 
    This is a useful argument type for arguments that will be passed into
 
    Books loader methods which can accept either a fiscal year or a full date.
 
    """
 
    try:
 
        year = int(arg, 10)
 
    except ValueError:
 
        ok = False
 
    else:
 
        ok = datetime.MINYEAR <= year <= datetime.MAXYEAR
 
    if ok:
 
        return year
 
    else:
 
        return date_arg(arg)
 

	
 
def make_entry_point(mod_name: str, prog_name: str=sys.argv[0]) -> Callable[[], int]:
 
    """Create an entry_point function for a tool
 

	
 
    The returned function is suitable for use as an entry_point in setup.py.
 
    It sets up the root logger and excepthook, then calls the module's main
 
    function.
 
    """
 
    def entry_point():  # type:ignore
 
        prog_mod = sys.modules[mod_name]
 
        setup_logger()
 
        prog_mod.logger = logging.getLogger(prog_name)
 
        sys.excepthook = ExceptHook(prog_mod.logger)
 
        return prog_mod.main()
 
    return entry_point
 

	
 
def setup_logger(logger: Union[str, logging.Logger]='',
 
                 stream: TextIO=sys.stderr,
 
                 fmt: str='%(name)s: %(levelname)s: %(message)s',
 
) -> logging.Logger:
 
    """Set up a logger with a StreamHandler with the given format"""
 
    if isinstance(logger, str):
 
        logger = logging.getLogger(logger)
 
    formatter = logging.Formatter(fmt)
 
    handler = logging.StreamHandler(stream)
 
    handler.setFormatter(formatter)
 
    logger.addHandler(handler)
 
    return logger
 

	
 
def set_loglevel(logger: logging.Logger, loglevel: int=logging.INFO) -> None:
 
    """Set the loglevel for a tool or module
 

	
 
    If the given logger is not under a hierarchy, this function sets the
 
    loglevel for the root logger, along with some specific levels for libraries
 
    used by reporting tools. Otherwise, it's the same as
 
    ``logger.setLevel(loglevel)``.
 
    """
 
    if '.' not in logger.name:
 
        logger = logging.getLogger()
 
        if loglevel <= logging.DEBUG:
 
            # At the debug level, the rt module logs the full body of every
 
            # request and response. That's too much.
 
            logging.getLogger('rt.rt').setLevel(logging.INFO)
 
    logger.setLevel(loglevel)
 

	
 
def bytes_output(path: Optional[Path]=None,
 
                 default: OutputFile=sys.stdout,
 
                 mode: str='w',
 
) -> BinaryIO:
 
    """Get a file-like object suitable for binary output
 

	
 
    If ``path`` is ``None`` or ``-``, returns a file-like object backed by
 
    ``default``. If ``default`` is a file descriptor or text IO object, this
 
    method returns a file-like object that writes to the same place.
 

	
 
    Otherwise, returns ``path.open(mode)``.
 
    """
 
    mode = f'{mode}b'
 
    if path is None or path == STDSTREAM_PATH:
 
        if isinstance(default, int):
 
            retval = open(default, mode)
 
        elif isinstance(default, TextIO):
 
            retval = default.buffer
 
        else:
 
            retval = default
 
    else:
 
        retval = path.open(mode)
 
    return cast(BinaryIO, retval)
 

	
 
def text_output(path: Optional[Path]=None,
 
                default: OutputFile=sys.stdout,
 
                mode: str='w',
 
                encoding: Optional[str]=None,
 
) -> TextIO:
 
    """Get a file-like object suitable for text output
 

	
 
    If ``path`` is ``None`` or ``-``, returns a file-like object backed by
 
    ``default``. If ``default`` is a file descriptor or binary IO object, this
 
    method returns a file-like object that writes to the same place.
 

	
 
    Otherwise, returns ``path.open(mode)``.
 
    """
 
    if path is None or path == STDSTREAM_PATH:
 
        if isinstance(default, int):
 
            retval = open(default, mode, encoding=encoding)
 
        elif isinstance(default, BinaryIO):
 
            retval = io.TextIOWrapper(default, encoding=encoding)
 
        else:
 
            retval = default
 
    else:
 
        retval = path.open(mode, encoding=encoding)
 
    return cast(TextIO, retval)
conservancy_beancount/reports/accrual.py
Show inline comments
 
#!/usr/bin/env python3
 
"""accrual-report - Status reports for accruals
 

	
 
accrual-report checks accruals (postings under Assets:Receivable and
 
Liabilities:Payable) for errors and metadata consistency, and reports any
 
problems on stderr. Then it writes a report about the status of those
 
accruals.
 

	
 
If you run it with no arguments, it will generate an aging report in ODS format.
 

	
 
Otherwise, the typical way to run it is to pass an RT ticket number or
 
invoice link as an argument, to report about accruals that match those
 
criteria::
 

	
 
    # Report all accruals associated with RT#1230:
 
    accrual-report 1230
 
    # Report all accruals with the invoice link rt:45/670.
 
    accrual-report 45/670
 
    # Report all accruals with the invoice link Invoice980.pdf.
 
    accrual-report Invoice980.pdf
 

	
 
By default, to stay fast, accrual-report only looks at unaudited books. You
 
can search further back in history by passing the ``--since`` argument. The
 
argument can be a fiscal year, or a negative number of how many years back
 
to search::
 

	
 
    # Search for accruals since 2016
 
    accrual-report --since 2016 [search terms …]
 
    # Search for accruals from the beginning of three fiscal years ago
 
    accrual-report --since -3 [search terms …]
 

	
 
If you want to further limit what accruals are reported, you can match on
 
other metadata by passing additional arguments in ``name=value`` format.
 
You can pass any number of search terms. For example::
 

	
 
    # Report accruals associated with RT#1230 and Jane Doe
 
    accrual-report 1230 entity=Doe-Jane
 

	
 
accrual-report will automatically decide what kind of report to generate
 
from the search terms you provide and the results they return. If you
 
searched on an RT ticket or invoice that returned a single outstanding
 
payable, it writes an outgoing approval report. If you searched on RT ticket
 
or invoice that returned other results, it writes a balance
 
report. Otherwise, it writes an aging report. You can specify what report
 
type you want with the ``--report-type`` option::
 

	
 
    # Write an outgoing approval report for all outstanding payables for
 
    # Jane Doe, even if there's more than one
 
    accrual-report --report-type outgoing entity=Doe-Jane
 
    # Write an aging report for a single RT invoice (this can be helpful when
 
    # one invoice covers multiple parties)
 
    accrual-report --report-type aging 12/345
 
"""
 
# Copyright © 2020  Brett Smith
 
#
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU Affero General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU Affero General Public License for more details.
 
#
 
# You should have received a copy of the GNU Affero General Public License
 
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 

	
 
import argparse
 
import collections
 
import datetime
 
import enum
 
import logging
 
import re
 
import sys
 

	
 
from pathlib import Path
 

	
 
from typing import (
 
    cast,
 
    Any,
 
    BinaryIO,
 
    Callable,
 
    Deque,
 
    Dict,
 
    Hashable,
 
    Iterable,
 
    Iterator,
 
    List,
 
    Mapping,
 
    Match,
 
    NamedTuple,
 
    Optional,
 
    Sequence,
 
    Set,
 
    TextIO,
 
    Tuple,
 
    TypeVar,
 
    Union,
 
)
 
from ..beancount_types import (
 
    Entries,
 
    Error,
 
    Errors,
 
    MetaKey,
 
    MetaValue,
 
    Transaction,
 
)
 

	
 
import odf.element  # type:ignore[import]
 
import odf.style  # type:ignore[import]
 
import odf.table  # type:ignore[import]
 
import rt
 

	
 
from beancount.parser import printer as bc_printer
 

	
 
from . import core
 
from . import rewrite
 
from .. import books
 
from .. import cliutil
 
from .. import config as configmod
 
from .. import data
 
from .. import filters
 
from .. import rtutil
 

	
 
PROGNAME = 'accrual-report'
 

	
 
PostGroups = Mapping[Optional[Hashable], 'AccrualPostings']
 
T = TypeVar('T')
 

	
 
logger = logging.getLogger('conservancy_beancount.reports.accrual')
 

	
 
class Account(NamedTuple):
 
    name: str
 
    aging_thresholds: Sequence[int]
 

	
 

	
 
class AccrualAccount(enum.Enum):
 
    # Note the aging report uses the same order accounts are defined here.
 
    # See AgingODS.start_spreadsheet().
 
    RECEIVABLE = Account('Assets:Receivable', [365, 120, 90, 60])
 
    PAYABLE = Account('Liabilities:Payable', [365, 90, 60, 30])
 

	
 
    @classmethod
 
    def account_names(cls) -> Iterator[str]:
 
        return (acct.value.name for acct in cls)
 

	
 
    @classmethod
 
    def by_account(cls, name: data.Account) -> 'AccrualAccount':
 
        for account in cls:
 
            if name.is_under(account.value.name):
 
                return account
 
        raise ValueError(f"unrecognized account {name!r}")
 

	
 
    @classmethod
 
    def classify(cls, related: core.RelatedPostings) -> 'AccrualAccount':
 
        for account in cls:
 
            account_name = account.value.name
 
            if all(post.account.is_under(account_name) for post in related):
 
                return account
 
        raise ValueError("unrecognized account set in related postings")
 

	
 
    @property
 
    def normalize_amount(self) -> Callable[[T], T]:
 
        return core.normalize_amount_func(self.value.name)
 

	
 

	
 
class AccrualPostings(core.RelatedPostings):
 
    __slots__ = ()
 

	
 
    @classmethod
 
    def make_consistent(cls,
 
                        postings: Iterable[data.Posting],
 
    ) -> Iterator[Tuple[Hashable, 'AccrualPostings']]:
 
        accruals: Dict[Tuple[str, ...], List[data.Posting]] = collections.defaultdict(list)
 
        payments: Dict[Tuple[str, ...], Deque[data.Posting]] = collections.defaultdict(Deque)
 
        key: Tuple[str, ...]
 
        for post in postings:
 
            norm_func = core.normalize_amount_func(post.account)
 
            invoice = str(post.meta.get('invoice', 'BlankInvoice'))
 
            if norm_func(post.units.number) >= 0:
 
                entity = str(post.meta.get('entity', 'BlankEntity'))
 
                key = (post.meta.date.isoformat(), entity, invoice, post.account)
 
                accruals[key].append(post)
 
            else:
 
                key = (invoice, post.account)
 
                payments[key].append(post)
 

	
 
        for key, acc_posts in accruals.items():
 
            pay_posts = payments[key[2:]]
 
            if not pay_posts:
 
                continue
 
            norm_func = core.normalize_amount_func(key[-1])
 
            balance = norm_func(core.MutableBalance(post.at_cost() for post in acc_posts))
 
            while pay_posts and not balance.le_zero():
 
                pay_post = pay_posts.popleft()
 
                acc_posts.append(pay_post)
 
                balance += norm_func(pay_post.at_cost())
 
            if balance.le_zero() and not balance.is_zero():
 
                # pay_post causes the accrual to be overpaid. Split it into two
 
                # synthesized postings: one that causes the accrual to be
 
                # exactly zero, and one with the remainder back in payments.
 
                post_cost = pay_post.at_cost()
 
                # Calling norm_func() reverses the call in the while loop to add
 
                # the amount to the balance.
 
                overpayment = norm_func(balance[post_cost.currency])
 
                amt_to_zero = post_cost._replace(number=post_cost.number - overpayment.number)
 
                acc_posts[-1] = pay_post._replace(units=amt_to_zero, cost=None, price=None)
 
                pay_posts.appendleft(pay_post._replace(units=overpayment, cost=None, price=None))
 
            acc_posts.sort(key=lambda post: post.meta.date)
 

	
 
        for key, acc_posts in accruals.items():
 
            yield key, cls(acc_posts, _can_own=True)
 
        for key, pay_posts in payments.items():
 
            if pay_posts:
 
                yield key, cls(pay_posts, _can_own=True)
 

	
 
    def is_paid(self) -> Optional[bool]:
 
        try:
 
            accrual_type = AccrualAccount.classify(self)
 
        except ValueError:
 
            return None
 
        else:
 
            return accrual_type.normalize_amount(self.balance()).le_zero()
 

	
 

	
 
class BaseReport:
 
    def __init__(self, out_file: TextIO) -> None:
 
        self.out_file = out_file
 
        self.logger = logger.getChild(type(self).__name__)
 

	
 
    def _report(self, posts: AccrualPostings, index: int) -> Iterable[str]:
 
        raise NotImplementedError("BaseReport._report")
 

	
 
    def run(self, groups: PostGroups) -> None:
 
        for index, invoice in enumerate(groups):
 
            for line in self._report(groups[invoice], index):
 
                print(line, file=self.out_file)
 

	
 

	
 
class AgingODS(core.BaseODS[AccrualPostings, data.Account]):
 
    AGE_COLORS = [
 
        '#ff00ff',
 
        '#ff0000',
 
        '#ff8800',
 
        '#ffff00',
 
        '#00ff00',
 
    ]
 
    DOC_COLUMNS = [
 
        'rt-id',
 
        'invoice',
 
        'approval',
 
        'contract',
 
        'purchase-order',
 
    ]
 
    COLUMNS = [
 
        'Date',
 
        data.Metadata.human_name('entity'),
 
        'Invoice Amount',
 
        'Booked Amount',
 
        data.Metadata.human_name('project'),
 
        *(data.Metadata.human_name(key) for key in DOC_COLUMNS),
 
    ]
 
    COL_COUNT = len(COLUMNS)
 

	
 
    def __init__(self,
 
                 rt_wrapper: rtutil.RT,
 
                 date: datetime.date,
 
                 logger: logging.Logger,
 
    ) -> None:
 
        super().__init__(rt_wrapper)
 
        self.date = date
 
        self.logger = logger
 

	
 
    def section_key(self, row: AccrualPostings) -> data.Account:
 
        return row[0].account
 

	
 
    def start_spreadsheet(self) -> None:
 
        for accrual_type in AccrualAccount:
 
            self.use_sheet(accrual_type.name.title())
 
            for index in range(self.COL_COUNT):
 
                if index == 0:
 
                    style: Union[str, odf.style.Style] = ''
 
                elif index < 6:
 
                    style = self.column_style(1.2)
 
                else:
 
                    style = self.column_style(1.5)
 
                self.sheet.addElement(odf.table.TableColumn(stylename=style))
 
            self.add_row(*(
 
                self.string_cell(name, stylename=self.style_bold)
 
                for name in self.COLUMNS
 
            ))
 
            self.lock_first_row()
 

	
 
    def start_section(self, key: data.Account) -> None:
 
        accrual_type = AccrualAccount.by_account(key)
 
        self.norm_func = accrual_type.normalize_amount
 
        self.age_thresholds = list(accrual_type.value.aging_thresholds)
 
        self.age_thresholds.append(-sys.maxsize)
 
        self.age_balances = [core.MutableBalance() for _ in self.age_thresholds]
 
        self.age_styles = [
 
            self.merge_styles(self.style_date, self.border_style(
 
                core.Border.LEFT, '10pt', 'solid', color,
 
            )) for color in self.AGE_COLORS
 
        ]
 
        acct_parts = key.slice_parts()
 
        self.use_sheet(acct_parts[1])
 
        self.add_row()
 
        self.add_row(self.string_cell(
 
            f"{' '.join(acct_parts[2:])} {acct_parts[1]} Aging Report"
 
            f" for {self.date.isoformat()}",
 
            stylename=self.merge_styles(self.style_bold, self.style_centertext),
 
            numbercolumnsspanned=self.COL_COUNT,
 
        ))
 
        self.add_row()
 

	
 
    def end_section(self, key: data.Account) -> None:
 
        total_balance = core.MutableBalance()
 
        text_span = 4
 
        last_age_text: Optional[str] = None
 
        self.add_row()
 
        for threshold, balance, style in zip(
 
                self.age_thresholds, self.age_balances, self.age_styles,
 
        ):
 
            years, days = divmod(threshold, 365)
 
            years_text = f"{years} {'Year' if years == 1 else 'Years'}"
 
            days_text = f"{days} Days"
 
            if years and days:
 
                age_text = f"{years_text} {days_text}"
 
            elif years:
 
                age_text = years_text
 
            else:
 
                age_text = days_text
 
            if last_age_text is None:
 
                age_range = f"Over {age_text}"
 
            elif threshold < 0:
 
                self.add_row(
 
                    self.string_cell(
 
                        f"Total Unpaid Over {last_age_text}: ",
 
                        stylename=self.merge_styles(self.style_bold, self.style_endtext),
 
                        numbercolumnsspanned=text_span,
 
                    ),
 
                    *(odf.table.TableCell() for _ in range(1, text_span)),
 
                    self.balance_cell(total_balance, stylename=self.style_total),
 
                )
 
                age_range = f"Under {last_age_text}"
 
            else:
 
                age_range = f"{age_text}–{last_age_text}"
 
            self.add_row(
 
                self.string_cell(
 
                    f"Total Aged {age_range}: ",
 
                    stylename=self.merge_styles(self.style_bold, self.style_endtext, style),
 
                    numbercolumnsspanned=text_span,
 
                ),
 
                *(odf.table.TableCell() for _ in range(1, text_span)),
 
                self.balance_cell(balance),
 
            )
 
            last_age_text = age_text
 
            total_balance += balance
 
        self.add_row(
 
            self.string_cell(
 
                "Total Unpaid: ",
 
                stylename=self.merge_styles(self.style_bold, self.style_endtext),
 
                numbercolumnsspanned=text_span,
 
            ),
 
            *(odf.table.TableCell() for _ in range(1, text_span)),
 
            self.balance_cell(total_balance, stylename=self.style_bottomline),
 
        )
 

	
 
    def write_row(self, row: AccrualPostings) -> None:
 
        row_date = row[0].meta.date
 
        row_balance = self.norm_func(row.balance_at_cost())
 
        age = (self.date - row_date).days
 
        for index, threshold in enumerate(self.age_thresholds):
 
            if age >= threshold:
 
                if row_balance.ge_zero():
 
                    self.age_balances[index] += row_balance
 
                break
 
        else:
 
            return
 
        raw_balance = self.norm_func(row.balance())
 
        if raw_balance == row_balance:
 
            amount_cell = odf.table.TableCell()
 
        else:
 
            amount_cell = self.balance_cell(raw_balance)
 
        entities = row.meta_values('entity')
 
        entities.discard(None)
 
        projects = row.meta_values('project')
 
        projects.discard(None)
 
        self.add_row(
 
            self.date_cell(row_date, stylename=self.age_styles[index]),
 
            self.multiline_cell(sorted(entities)),
 
            amount_cell,
 
            self.balance_cell(row_balance),
 
            self.multiline_cell(sorted(projects)),
 
            *(self.meta_links_cell(row.all_meta_links(key))
 
              for key in self.DOC_COLUMNS),
 
        )
 

	
 

	
 
class AgingReport(BaseReport):
 
    def __init__(self,
 
                 rt_wrapper: rtutil.RT,
 
                 out_file: BinaryIO,
 
                 date: Optional[datetime.date]=None,
 
    ) -> None:
 
        if date is None:
 
            date = datetime.date.today()
 
        self.out_bin = out_file
 
        self.logger = logger.getChild(type(self).__name__)
 
        self.ods = AgingODS(rt_wrapper, date, self.logger)
 

	
 
    def run(self, groups: PostGroups) -> None:
 
        rows = [group for group in groups.values()
 
                if not group.balance_at_cost().is_zero()]
 
        rows.sort(key=lambda group: (
 
            group[0].account,
 
            group[0].meta.date,
 
            abs(sum(amt.number for amt in group.balance_at_cost().values())),
 
        ))
 
        self.ods.write(rows)
 
        self.ods.save_file(self.out_bin)
 

	
 

	
 
class BalanceReport(BaseReport):
 
    def _report(self, posts: AccrualPostings, index: int) -> Iterable[str]:
 
        meta = posts[0].meta
 
        date_s = meta.date.strftime('%Y-%m-%d')
 
        entity_s = meta.get('entity', '<no entity>')
 
        invoice_s = meta.get('invoice', '<no invoice>')
 
        balance_s = posts.balance_at_cost().format(zero="Zero balance")
 
        if index:
 
            yield ""
 
        yield f"{entity_s} {invoice_s}:"
 
        yield f"  {balance_s} outstanding since {date_s}"
 

	
 

	
 
class OutgoingReport(BaseReport):
 
    class PaymentMethods(enum.Enum):
 
        ach = 'ACH'
 
        check = 'Check'
 
        creditcard = 'Credit Card'
 
        credit_card = creditcard
 
        debitcard = 'Debit Card'
 
        debit_card = debitcard
 
        echeck = 'E-Check'
 
        e_check = echeck
 
        paypal = 'PayPal'
 
        pay_pal = paypal
 
        vendorportal = 'Vendor Portal'
 
        vendor_portal = vendorportal
 
        wire = 'Wire'
 
        fxwire = wire
 
        fx_wire = fxwire
 
        uswire = wire
 
        us_wire = uswire
 

	
 

	
 
    def __init__(self, rt_wrapper: rtutil.RT, out_file: TextIO) -> None:
 
        super().__init__(out_file)
 
        self.rt_wrapper = rt_wrapper
 
        self.rt_client = rt_wrapper.rt
 

	
 
    def _primary_rt_id(self, posts: AccrualPostings) -> rtutil.TicketAttachmentIds:
 
        rt_ids = posts.first_meta_links('rt-id')
 
        rt_id = next(rt_ids, None)
 
        rt_id2 = next(rt_ids, None)
 
        if rt_id is None:
 
            raise ValueError("no rt-id links found")
 
        elif rt_id2 is not None:
 
            raise ValueError("multiple rt-id links found")
 
        parsed = rtutil.RT.parse(rt_id)
 
        if parsed is None:
 
            raise ValueError("rt-id is not a valid RT reference")
 
        else:
 
            return parsed
 

	
 
    def _get_payment_method(self, posts: AccrualPostings, ticket_id: str) -> Optional[str]:
 
        payment_methods = posts.meta_values('payment-method')
 
        payment_methods.discard(None)
 
        if all(isinstance(s, str) for s in payment_methods):
 
            # type ignore for <https://github.com/python/mypy/issues/7853>
 
            payment_methods = {s.strip().lower() for s in payment_methods}  # type:ignore[union-attr]
 
        log_prefix = f"cannot set payment-method for rt:{ticket_id}:"
 
        payment_method_count = len(payment_methods)
 
        if payment_method_count != 1:
 
            self.logger.warning("%s %s metadata values found",
 
                                log_prefix, payment_method_count)
 
            return None
 
        payment_method = payment_methods.pop()
 
        if not isinstance(payment_method, str):
 
            self.logger.warning("%s %r is not a string value",
 
                                log_prefix, payment_method)
 
            return None
 
        try:
 
            currency, method_key = payment_method.split(None, 1)
 
        except ValueError:
 
            self.logger.warning("%s no method specified in %r",
 
                                log_prefix, payment_method)
 
            return None
 
        curr_match = re.fullmatch(r'[a-z]{3}', currency)
 
        if curr_match is None:
 
            self.logger.warning("%s invalid currency %r",
 
                                log_prefix, currency)
 
        try:
 
            method_enum = self.PaymentMethods[re.sub(r'[- ]', '_', method_key)]
 
        except KeyError:
 
            self.logger.warning("%s invalid method %r",
 
                                log_prefix, method_key)
 
            curr_match = None
 
        if curr_match is None:
 
            return None
 
        else:
 
            return f'{currency.upper()} {method_enum.value}'
 

	
 
    def _report(self, posts: AccrualPostings, index: int) -> Iterable[str]:
 
        try:
 
            ticket_id, _ = self._primary_rt_id(posts)
 
            ticket = self.rt_client.get_ticket(ticket_id)
 
            # Note we only use this when ticket is None.
 
            errmsg = f"ticket {ticket_id} not found"
 
        except (ValueError, rt.RtError) as error:
 
            ticket = None
 
            errmsg = error.args[0]
 
        if ticket is None:
 
            meta = posts[0].meta
 
            self.logger.error(
 
                "can't generate outgoings report for %s %s %s because no RT ticket available: %s",
 
                meta.date.isoformat(),
 
                meta.get('entity', '<no entity>'),
 
                meta.get('invoice', '<no invoice>'),
 
                errmsg,
 
            )
 
            return
 

	
 
        try:
 
            rt_requestor = self.rt_client.get_user(ticket['Requestors'][0])
 
        except (IndexError, rt.RtError):
 
            rt_requestor = None
 
        if rt_requestor is None:
 
            requestor = ''
 
            requestor_name = ''
 
        else:
 
            requestor_name = (
 
                rt_requestor.get('RealName')
 
                or ticket.get('CF.{payment-to}')
 
                or ''
 
            )
 
            requestor = f'{requestor_name} <{rt_requestor["EmailAddress"]}>'.strip()
 

	
 
        last_zero_index = -1
 
        for index, (post, balance) in enumerate(posts.iter_with_balance()):
 
            if balance.is_zero():
 
                prior_zero_index = last_zero_index
 
                last_zero_index = index
 
        if last_zero_index == index:
 
            last_zero_index = prior_zero_index
 
        posts = posts[last_zero_index + 1:]
 

	
 
        balance = -posts.balance_at_cost()
 
        balance_s = balance.format(None)
 
        raw_balance = -posts.balance()
 
        payment_amount = raw_balance.format('¤¤ #,##0.00')
 
        if raw_balance != balance:
 
            payment_amount += f' ({balance_s})'
 
            balance_s = f'{raw_balance} ({balance_s})'
 

	
 
        payment_to = ticket.get('CF.{payment-to}') or requestor_name
 
        contract_links = list(posts.all_meta_links('contract'))
 
        if contract_links:
 
            contract_s = ' , '.join(self.rt_wrapper.iter_urls(
 
                contract_links, missing_fmt='<BROKEN RT LINK: {}>',
 
            ))
 
        else:
 
            contract_s = "NO CONTRACT GOVERNS THIS TRANSACTION"
 
        projects = [v for v in posts.meta_values('project')
 
                    if isinstance(v, str)]
 

	
 
        yield "PAYMENT FOR APPROVAL:"
 
        yield f"REQUESTOR: {requestor}"
 
        yield f"PAYMENT TO: {payment_to}"
 
        yield f"TOTAL TO PAY: {balance_s}"
 
        yield f"AGREEMENT: {contract_s}"
 
        yield f"PROJECT: {', '.join(projects)}"
 
        yield "\nBEANCOUNT ENTRIES:\n"
 

	
 
        last_txn: Optional[Transaction] = None
 
        for post in posts:
 
            txn = post.meta.txn
 
            if txn is not last_txn:
 
                last_txn = txn
 
                txn = self.rt_wrapper.txn_with_urls(txn, '{}')
 
                # Suppress payment-method metadata from the report.
 
                txn.meta.pop('payment-method', None)
 
                for txn_post in txn.postings:
 
                    if txn_post.meta:
 
                        txn_post.meta.pop('payment-method', None)
 
                yield bc_printer.format_entry(txn)
 

	
 
        cf_targets = {
 
            'payment-amount': payment_amount,
 
            'payment-method': (self._get_payment_method(posts, ticket_id)
 
                               or ticket.get('CF.{payment-method}')),
 
            'payment-to': payment_to,
 
        }
 

	
 
        cf_updates = {
 
            f'CF_{key}': value
 
            for key, value in cf_targets.items()
 
            if ticket.get(f'CF.{{{key}}}') != value
 
        }
 
        if cf_updates:
 
            try:
 
                ok = self.rt_client.edit_ticket(ticket_id, **cf_updates)
 
            except rt.RtError:
 
                self.logger.debug("RT exception on edit_ticket", exc_info=True)
 
                ok = False
 
            if not ok:
 
                self.logger.warning("failed to set custom fields for rt:%s", ticket_id)
 

	
 

	
 
class ReportType(enum.Enum):
 
    AGING = AgingReport
 
    BALANCE = BalanceReport
 
    OUTGOING = OutgoingReport
 
    AGE = AGING
 
    BAL = BALANCE
 
    OUT = OUTGOING
 
    OUTGOINGS = OUTGOING
 

	
 
    @classmethod
 
    def by_name(cls, name: str) -> 'ReportType':
 
        try:
 
            return cls[name.upper()]
 
        except KeyError:
 
            raise ValueError(f"unknown report type {name!r}") from None
 

	
 

	
 
def filter_search(postings: Iterable[data.Posting],
 
                  search_terms: Iterable[cliutil.SearchTerm],
 
) -> Iterable[data.Posting]:
 
    accounts = tuple(AccrualAccount.account_names())
 
    postings = (post for post in postings if post.account.is_under(*accounts))
 
    for query in search_terms:
 
        postings = query.filter_postings(postings)
 
    return postings
 

	
 
def parse_arguments(arglist: Optional[Sequence[str]]=None) -> argparse.Namespace:
 
    parser = argparse.ArgumentParser(prog=PROGNAME)
 
    cliutil.add_version_argument(parser)
 
    cliutil.add_rewrite_rules_argument(parser)
 
    parser.add_argument(
 
        '--report-type', '-t',
 
        metavar='NAME',
 
        type=ReportType.by_name,
 
        help="""The type of report to generate, one of `aging`, `balance`, or
 
`outgoing`. If not specified, the default is `aging` when no search terms are
 
given, `outgoing` for search terms that return a single outstanding payable,
 
and `balance` any other time.
 
""")
 
    parser.add_argument(
 
        '--since',
 
        metavar='YEAR',
 
        type=int,
 
        default=0,
 
        help="""How far back to search the books for related transactions.
 
You can either specify a fiscal year, or a negative offset from the current
 
fiscal year, to start loading entries from. The default is to load the current,
 
unaudited books.
 
""")
 
    parser.add_argument(
 
        '--output-file', '-O',
 
        metavar='PATH',
 
        type=Path,
 
        help="""Write the report to this file, or stdout when PATH is `-`.
 
The default is stdout for the balance and outgoing reports, and a generated
 
filename for other reports.
 
""")
 
    cliutil.add_loglevel_argument(parser)
 
    parser.add_argument(
 
        'search_terms',
 
        metavar='FILTER',
 
        type=cliutil.SearchTerm.arg_parser('invoice', 'rt-id'),
 
        nargs=argparse.ZERO_OR_MORE,
 
        help="""Report on accruals that match this criteria. The format is
 
NAME=TERM. TERM is a link or word that must exist in a posting's NAME
 
metadata to match. A single ticket number is a shortcut for
 
`rt-id=rt:NUMBER`. Any other link, including an RT attachment link in
 
`TIK/ATT` format, is a shortcut for `invoice=LINK`.
 
""")
 
    args = parser.parse_args(arglist)
 
    if args.report_type is None and not any(
 
            term.meta_key == 'invoice' or term.meta_key == 'rt-id'
 
            for term in args.search_terms
 
    ):
 
        args.report_type = ReportType.AGING
 
    return args
 

	
 
def main(arglist: Optional[Sequence[str]]=None,
 
         stdout: TextIO=sys.stdout,
 
         stderr: TextIO=sys.stderr,
 
         config: Optional[configmod.Config]=None,
 
) -> int:
 
    args = parse_arguments(arglist)
 
    cliutil.set_loglevel(logger, args.loglevel)
 
    if config is None:
 
        config = configmod.Config()
 
        config.load_file()
 

	
 
    returncode = 0
 
    books_loader = config.books_loader()
 
    if books_loader is None:
 
        entries, load_errors, _ = books.Loader.load_none(config.config_file_path())
 
        returncode = cliutil.ExitCode.NoConfiguration
 
    else:
 
        load_since = None if args.report_type == ReportType.AGING else args.since
 
        entries, load_errors, _ = books_loader.load_all(load_since)
 
        if load_errors:
 
            returncode = cliutil.ExitCode.BeancountErrors
 
        elif not entries:
 
            returncode = cliutil.ExitCode.NoDataLoaded
 
    filters.remove_opening_balance_txn(entries)
 
    for error in load_errors:
 
        bc_printer.print_error(error, file=stderr)
 

	
 
    postings = list(filter_search(
 
        data.Posting.from_entries(entries), args.search_terms,
 
    ))
 
    postings_src = data.Posting.from_entries(entries)
 
    for rewrite_path in args.rewrite_rules:
 
        try:
 
            ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path)
 
        except ValueError as error:
 
            logger.critical("failed loading rewrite rules from %s: %s",
 
                            rewrite_path, error.args[0])
 
            return cliutil.ExitCode.RewriteRulesError
 
        postings_src = ruleset.rewrite(postings_src)
 
    postings = list(filter_search(postings_src, args.search_terms))
 
    if not postings:
 
        logger.warning("no matching entries found to report")
 
        returncode = returncode or cliutil.ExitCode.NoDataFiltered
 
    # groups is a mapping of metadata value strings to AccrualPostings.
 
    # The keys are basically arbitrary, the report classes don't rely on them,
 
    # but they do help symbolize what's being grouped.
 
    # For the outgoing approval report, groups maps rt-id link strings to
 
    # associated accruals.
 
    # For all other reports, groups comes from AccrualReport.make_consistent().
 
    groups: PostGroups
 
    if args.report_type is None or args.report_type is ReportType.OUTGOING:
 
        groups = dict(AccrualPostings.group_by_first_meta_link(postings, 'rt-id'))
 
        if args.report_type is None and len(groups) == 1:
 
            key = next(iter(groups))
 
            group = groups[key]
 
            account = group[0].account
 
            if (AccrualAccount.by_account(account) is AccrualAccount.PAYABLE
 
                and all(post.account == account for post in group)
 
                and not group.balance().ge_zero()
 
                and key):  # Make sure we have a usable rt-id
 
                args.report_type = ReportType.OUTGOING
 
    if args.report_type is not ReportType.OUTGOING:
 
        groups = dict(AccrualPostings.make_consistent(postings))
 
    if args.report_type is not ReportType.AGING:
 
        groups = {
 
            key: posts for key, posts in groups.items() if not posts.is_paid()
 
        } or groups
 
    del postings
 

	
 
    report: Optional[BaseReport] = None
 
    output_path: Optional[Path] = None
 
    if args.report_type is ReportType.AGING:
 
        rt_wrapper = config.rt_wrapper()
 
        if rt_wrapper is None:
 
            logger.error("unable to generate aging report: RT client is required")
 
        else:
 
            if args.output_file is None:
 
                now = datetime.datetime.now()
 
                out_dir_path = config.repository_path() or Path()
 
                args.output_file = out_dir_path / now.strftime('AgingReport_%Y-%m-%d_%H:%M.ods')
 
                logger.info("Writing report to %s", args.output_file)
 
            out_bin = cliutil.bytes_output(args.output_file, stdout)
 
            report = AgingReport(rt_wrapper, out_bin)
 
            report.ods.set_common_properties(config.books_repo())
 
    elif args.report_type is ReportType.OUTGOING:
 
        rt_wrapper = config.rt_wrapper()
 
        if rt_wrapper is None:
 
            logger.error("unable to generate outgoing report: RT client is required")
 
        else:
 
            out_file = cliutil.text_output(args.output_file, stdout)
 
            report = OutgoingReport(rt_wrapper, out_file)
 
    else:
 
        out_file = cliutil.text_output(args.output_file, stdout)
 
        report = BalanceReport(out_file)
 

	
 
    if report is None:
 
        returncode = cliutil.ExitCode.NoConfiguration
 
    else:
 
        report.run(groups)
 
    return returncode
 

	
 
entry_point = cliutil.make_entry_point(__name__, PROGNAME)
 

	
 
if __name__ == '__main__':
 
    exit(entry_point())
conservancy_beancount/reports/balance_sheet.py
Show inline comments
 
"""balance_sheet.py - Balance sheet report"""
 
# Copyright © 2020  Brett Smith
 
#
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU Affero General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU Affero General Public License for more details.
 
#
 
# You should have received a copy of the GNU Affero General Public License
 
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 

	
 
import argparse
 
import collections
 
import datetime
 
import enum
 
import logging
 
import operator
 
import os
 
import sys
 

	
 
from decimal import Decimal
 
from pathlib import Path
 

	
 
from typing import (
 
    Any,
 
    Callable,
 
    Collection,
 
    Dict,
 
    Hashable,
 
    Iterable,
 
    Iterator,
 
    List,
 
    Mapping,
 
    NamedTuple,
 
    Optional,
 
    Sequence,
 
    TextIO,
 
    Tuple,
 
    Union,
 
)
 

	
 
import odf.style  # type:ignore[import]
 
import odf.table  # type:ignore[import]
 

	
 
from beancount.parser import printer as bc_printer
 

	
 
from . import core
 
from . import rewrite
 
from .. import books
 
from .. import cliutil
 
from .. import config as configmod
 
from .. import data
 
from .. import ranges
 

	
 
EQUITY_ACCOUNTS = frozenset(['Equity', 'Income', 'Expenses'])
 
PROGNAME = 'balance-sheet-report'
 
logger = logging.getLogger('conservancy_beancount.reports.balance_sheet')
 

	
 
KWArgs = Mapping[str, Any]
 

	
 
class Fund(enum.IntFlag):
 
    RESTRICTED = enum.auto()
 
    UNRESTRICTED = enum.auto()
 
    ANY = RESTRICTED | UNRESTRICTED
 

	
 

	
 
class Period(enum.IntFlag):
 
    OPENING = enum.auto()
 
    PRIOR = enum.auto()
 
    MIDDLE = enum.auto()
 
    PERIOD = enum.auto()
 
    THRU_PRIOR = OPENING | PRIOR
 
    THRU_MIDDLE = THRU_PRIOR | MIDDLE
 
    ANY = THRU_MIDDLE | PERIOD
 

	
 

	
 
class BalanceKey(NamedTuple):
 
    account: data.Account
 
    classification: data.Account
 
    period: Period
 
    fund: Fund
 
    post_type: Optional[str]
 

	
 

	
 
class Balances:
 
    def __init__(self,
 
                 postings: Iterable[data.Posting],
 
                 start_date: datetime.date,
 
                 stop_date: datetime.date,
 
                 fund_key: str='project',
 
                 unrestricted_fund_value: str='Conservancy',
 
    ) -> None:
 
        year_diff = (stop_date - start_date).days // 365
 
        if year_diff == 0:
 
            self.prior_range = ranges.DateRange(
 
                cliutil.diff_year(start_date, -1),
 
                cliutil.diff_year(stop_date, -1),
 
            )
 
            self.period_desc = "Period"
 
        else:
 
            self.prior_range = ranges.DateRange(
 
                cliutil.diff_year(start_date, -year_diff),
 
                start_date,
 
            )
 
            self.period_desc = f"Year{'s' if year_diff > 1 else ''}"
 
        self.middle_range = ranges.DateRange(self.prior_range.stop, start_date)
 
        self.period_range = ranges.DateRange(start_date, stop_date)
 
        self.balances: Mapping[BalanceKey, core.MutableBalance] \
 
            = collections.defaultdict(core.MutableBalance)
 
        for post in postings:
 
            post_date = post.meta.date
 
            if post_date >= stop_date:
 
                continue
 
            elif post_date in self.period_range:
 
                period = Period.PERIOD
 
            elif post_date in self.middle_range:
 
                period = Period.MIDDLE
 
            elif post_date in self.prior_range:
 
                period = Period.PRIOR
 
            else:
 
                period = Period.OPENING
 
            if post.meta.get(fund_key) == unrestricted_fund_value:
 
                fund = Fund.UNRESTRICTED
 
            else:
 
                fund = Fund.RESTRICTED
 
            try:
 
                classification_s = post.account.meta['classification']
 
                if isinstance(classification_s, str):
 
                    classification = data.Account(classification_s)
 
                else:
 
                    raise TypeError()
 
            except (KeyError, TypeError):
 
                classification = post.account
 
            if post.account.root_part() == 'Expenses':
 
                post_type = post.meta.get('expense-type')
 
            else:
 
                post_type = None
 
            key = BalanceKey(post.account, classification, period, fund, post_type)
 
            self.balances[key] += post.at_cost()
 

	
 
    def total(self,
 
              account: Union[None, str, Collection[str]]=None,
 
              classification: Optional[str]=None,
 
              period: int=Period.ANY,
 
              fund: int=Fund.ANY,
 
              post_type: Optional[str]=None,
 
    ) -> core.Balance:
 
        if isinstance(account, str):
 
            account = (account,)
 
        retval = core.MutableBalance()
 
        for key, balance in self.balances.items():
 
            if not (account is None or key.account.is_under(*account)):
 
                pass
 
            elif not (classification is None
 
                      or key.classification.is_under(classification)):
 
                pass
 
            elif not period & key.period:
 
                pass
 
            elif not fund & key.fund:
 
                pass
 
            elif not (post_type is None or post_type == key.post_type):
 
                pass
 
            else:
 
                retval += balance
 
        return retval
 

	
 
    def classifications(self,
 
                        account: str,
 
                        sort_period: Optional[int]=None,
 
    ) -> Sequence[data.Account]:
 
        if sort_period is None:
 
            if account in EQUITY_ACCOUNTS:
 
                sort_period = Period.PERIOD
 
            else:
 
                sort_period = Period.ANY
 
        class_bals: Mapping[data.Account, core.MutableBalance] \
 
            = collections.defaultdict(core.MutableBalance)
 
        for key, balance in self.balances.items():
 
            if not key.account.is_under(account):
 
                pass
 
            elif key.period & sort_period:
 
                class_bals[key.classification] += balance
 
            else:
 
                # Ensure the balance exists in the mapping
 
                class_bals[key.classification]
 
        norm_func = core.normalize_amount_func(f'{account}:RootsOK')
 
        def sortkey(acct: data.Account) -> Hashable:
 
            prefix, _, _ = acct.rpartition(':')
 
            balance = norm_func(class_bals[acct])
 
            try:
 
                max_bal = max(amount.number for amount in balance.values())
 
            except ValueError:
 
                max_bal = Decimal(0)
 
            return prefix, -max_bal
 
        return sorted(class_bals, key=sortkey)
 

	
 
    def iter_accounts(self, root: str) -> Sequence[data.Account]:
 
        start_date = self.period_range.start
 
        stop_date = self.period_range.stop
 
        return sorted(
 
            account
 
            for account in data.Account.iter_accounts(root)
 
            if account.meta.open_date < stop_date
 
            and (account.meta.close_date is None
 
                 or account.meta.close_date > start_date)
 
        )
 

	
 

	
 
class Report(core.BaseODS[Sequence[None], None]):
 
    C_CASH = 'Cash'
 
    NO_BALANCE = core.Balance()
 
    SPACE = ' ' * 4
 

	
 
    def __init__(self,
 
                 balances: Balances,
 
                 *,
 
                 date_fmt: str='%B %d, %Y',
 
    ) -> None:
 
        super().__init__()
 
        self.balances = balances
 
        self.period_desc = balances.period_desc
 
        self.date_fmt = date_fmt
 
        one_day = datetime.timedelta(days=1)
 
        date = balances.period_range.stop - one_day
 
        self.period_name = date.strftime(date_fmt)
 
        date = balances.prior_range.stop - one_day
 
        self.opening_name = date.strftime(date_fmt)
 
        self.last_totals_row = odf.table.TableRow()
 

	
 
    def section_key(self, row: Sequence[None]) -> None:
 
        raise NotImplementedError("balance_sheet.Report.section_key")
 

	
 
    def init_styles(self) -> None:
 
        super().init_styles()
 
        self.style_header = self.merge_styles(self.style_bold, self.style_centertext)
 
        self.style_huline = self.merge_styles(
 
            self.style_header,
 
            self.border_style(core.Border.BOTTOM, '1pt'),
 
        )
 

	
 
    def write_all(self) -> None:
 
        self.write_financial_position()
 
        self.write_activities()
 
        self.write_functional_expenses()
 
        self.write_cash_flows()
 
        self.write_trial_balances()
 

	
 
    def walk_classifications(self, cseq: Iterable[data.Account]) \
 
        -> Iterator[Tuple[str, Optional[data.Account]]]:
 
        last_prefix: Sequence[str] = []
 
        for classification in cseq:
 
            parts = classification.split(':')
 
            tail = parts.pop()
 
            space = self.SPACE * len(parts)
 
            if parts != last_prefix:
 
                yield f'{space[len(self.SPACE):]}{parts[-1]}', None
 
                last_prefix = parts
 
            yield f'{space}{tail}', classification
 

	
 
    def walk_classifications_by_account(
 
            self,
 
            account: str,
 
            sort_period: Optional[int]=None,
 
    ) -> Iterator[Tuple[str, Optional[data.Account]]]:
 
        return self.walk_classifications(self.balances.classifications(
 
            account, sort_period,
 
        ))
 

	
 
    def start_sheet(self,
 
                    sheet_name: str,
 
                    *headers: Iterable[str],
 
                    totals_prefix: Sequence[str]=(),
 
                    first_width: Union[float, str]=3,
 
                    width: Union[float, str]=1.5,
 
                    title_fmt: str="DRAFT Statement of {sheet_name}",
 
    ) -> None:
 
        header_cells: Sequence[odf.table.TableCell] = [
 
            odf.table.TableCell(),
 
            *(self.multiline_cell(header_lines, stylename=self.style_huline)
 
              for header_lines in headers),
 
            *(self.multiline_cell([*totals_prefix, date_s], stylename=self.style_huline)
 
              for date_s in [self.period_name, self.opening_name]),
 
        ]
 
        self.col_count = len(header_cells)
 
        self.use_sheet(sheet_name)
 
        for index in range(self.col_count):
 
            col_style = self.column_style(width if index else first_width)
 
            self.sheet.addElement(odf.table.TableColumn(stylename=col_style))
 
        start_date = self.balances.period_range.start.strftime(self.date_fmt)
 
        self.add_row(
 
            self.multiline_cell([
 
                title_fmt.format(sheet_name=sheet_name),
 
                f"{start_date}—{self.period_name}",
 
            ], numbercolumnsspanned=self.col_count, stylename=self.style_header)
 
        )
 
        self.add_row()
 
        self.add_row(*header_cells)
 

	
 
    def write_classifications_by_account(
 
            self,
 
            account: str,
 
            balance_kwargs: Sequence[KWArgs],
 
            exclude_classifications: Collection[str]=frozenset(),
 
            text_prefix: str='',
 
            norm_func: Optional[Callable[[core.Balance], core.Balance]]=None,
 
    ) -> Sequence[core.Balance]:
 
        if norm_func is None:
 
            norm_func = core.normalize_amount_func(f'{account}:RootsOK')
 
        assert len(balance_kwargs) + 1 == self.col_count, \
 
            "called write_classifications with wrong number of balance_kwargs"
 
        retval = [core.MutableBalance() for _ in balance_kwargs]
 
        for text, classification in self.walk_classifications_by_account(account):
 
            text_cell = self.string_cell(text_prefix + text)
 
            if classification is None:
 
                if not text[0].isspace():
 
                    self.add_row()
 
                self.add_row(text_cell)
 
            elif classification in exclude_classifications:
 
                pass
 
            else:
 
                row = self.add_row(text_cell)
 
                for kwargs, total_bal in zip(balance_kwargs, retval):
 
                    balance = norm_func(self.balances.total(
 
                        classification=classification, **kwargs,
 
                    ))
 
                    row.addElement(self.balance_cell(balance))
 
                    total_bal += balance
 
        return retval
 

	
 
    def write_totals_row(
 
            self,
 
            text: str,
 
            *balances: Sequence[core.Balance],
 
            stylename: Union[None, str, odf.style.Style]=None,
 
            leading_rows: Optional[int]=None,
 
    ) -> odf.table.TableRow:
 
        if leading_rows is None:
 
            if (self.sheet.lastChild is self.last_totals_row
 
                or stylename is self.style_bottomline):
 
                leading_rows = 1
 
            else:
 
                leading_rows = 0
 
        expect_len = self.col_count - 1
 
        assert all(len(seq) == expect_len for seq in balances), \
 
            "called write_totals_row with the wrong length of balance columns"
 
        for _ in range(leading_rows):
 
            self.add_row()
 
        self.last_totals_row = self.add_row(
 
            self.string_cell(text),
 
            *(self.balance_cell(
 
                sum(sum_bals, core.MutableBalance()),
 
                stylename=stylename,
 
            ) for sum_bals in zip(*balances)),
 
        )
 
        return self.last_totals_row
 

	
 
    def write_financial_position(self) -> None:
 
        self.start_sheet("Financial Position")
 
        balance_kwargs: Sequence[KWArgs] = [
 
            {'period': Period.ANY},
 
            {'period': Period.THRU_PRIOR},
 
        ]
 

	
 
        asset_totals = self.write_classifications_by_account('Assets', balance_kwargs)
 
        self.write_totals_row(
 
            "Total Assets", asset_totals, stylename=self.style_bottomline,
 
        )
 
        self.add_row()
 
        self.add_row()
 

	
 
        liabilities = self.write_classifications_by_account('Liabilities', balance_kwargs)
 
        self.write_totals_row(
 
            "Total Liabilities", liabilities, stylename=self.style_endtotal,
 
        )
 
        self.add_row()
 
        self.add_row()
 

	
 
        equity_totals = [core.MutableBalance() for _ in balance_kwargs]
 
        self.add_row(self.string_cell("Net Assets", stylename=self.style_bold))
 
        self.add_row()
 
        for fund in [Fund.UNRESTRICTED, Fund.RESTRICTED]:
 
            preposition = "Without" if fund is Fund.UNRESTRICTED else "With"
 
            row = self.add_row(self.string_cell(f"{preposition} donor restrictions"))
 
            for kwargs, total_bal in zip(balance_kwargs, equity_totals):
 
                balance = -self.balances.total(account=EQUITY_ACCOUNTS, fund=fund, **kwargs)
 
                row.addElement(self.balance_cell(balance))
 
                total_bal += balance
 
        self.write_totals_row(
 
            "Total Net Assets", equity_totals, stylename=self.style_total,
 
        )
 
        self.write_totals_row(
 
            "Total Liabilities and Net Assets",
 
            liabilities, equity_totals,
 
            stylename=self.style_bottomline,
 
        )
 

	
 
    def write_activities(self) -> None:
 
        self.start_sheet(
 
            "Activities",
 
            ["Without Donor", "Restrictions"],
 
            ["With Donor", "Restrictions"],
 
            totals_prefix=[f"Total {self.period_desc} Ended"],
 
        )
 
        bal_kwargs: Sequence[Dict[str, Any]] = [
 
            {'period': Period.PERIOD, 'fund': Fund.UNRESTRICTED},
 
            {'period': Period.PERIOD, 'fund': Fund.RESTRICTED},
 
            {'period': Period.PERIOD},
 
            {'period': Period.PRIOR},
 
        ]
 

	
 
        self.add_row(self.string_cell("Support and Revenue", stylename=self.style_bold))
 
        self.add_row()
 
        income_totals = self.write_classifications_by_account('Income', bal_kwargs)
 
        self.write_totals_row("", income_totals, stylename=self.style_total)
 
        self.add_row()
 
        self.add_row(
 
            self.string_cell("Net Assets released from restrictions:"),
 
        )
 
        released = self.balances.total(
 
            account=('Expenses', 'Equity'),
 
            period=Period.PERIOD,
 
            fund=Fund.RESTRICTED,
 
        )
 
        other_totals = [core.MutableBalance() for _ in bal_kwargs]
 
        other_totals[0] += released
 
        other_totals[1] -= released
 
        self.write_totals_row("Satisfaction of program restrictions", other_totals)
 
        self.write_totals_row(
 
            "Total Support and Revenue",
 
            income_totals, other_totals,
 
            stylename=self.style_endtotal,
 
        )
 

	
 
        period_expenses = core.MutableBalance()
 
        prior_expenses = core.MutableBalance()
 
        self.add_row()
 
        self.add_row(self.string_cell("Expenses", stylename=self.style_bold))
 
        self.add_row()
 
        for text, type_value in [
 
                ("Program services", 'program'),
 
                ("Management and administrative services", 'management'),
 
                ("Fundraising", 'fundraising'),
 
        ]:
 
            period_bal = self.balances.total(
 
                account='Expenses', period=Period.PERIOD, post_type=type_value,
 
            )
 
            prior_bal = self.balances.total(
 
                account='Expenses', period=Period.PRIOR, post_type=type_value,
 
            )
 
            self.write_totals_row(text, [
 
                period_bal,
 
                self.NO_BALANCE,
 
                period_bal,
 
                prior_bal,
 
            ], leading_rows=0)
 
            period_expenses += period_bal
 
            prior_expenses += prior_bal
 
        period_bal = self.balances.total(account='Expenses', period=Period.PERIOD)
 
        if (period_expenses - period_bal).clean_copy(1).is_zero():
 
            period_bal = period_expenses
 
        else:
 
            logger.warning("Period functional expenses do not match total; math in columns B+D is wrong")
 
        prior_bal = self.balances.total(account='Expenses', period=Period.PRIOR)
 
        if (prior_expenses - prior_bal).clean_copy(1).is_zero():
 
            prior_bal = prior_expenses
 
        else:
 
            logger.warning("Prior functional expenses do not match total; math in column E is wrong")
 
        self.write_totals_row("Total Expenses", [
 
            period_bal,
 
            self.NO_BALANCE,
 
            period_bal,
 
            prior_bal,
 
        ], stylename=self.style_endtotal, leading_rows=0)
 

	
 
        other_totals[0] -= period_bal
 
        other_totals[2] -= period_bal
 
        other_totals[3] -= prior_bal
 
        self.write_totals_row("Change in Net Assets", income_totals, other_totals)
 

	
 
        for kwargs in bal_kwargs:
 
            if kwargs['period'] is Period.PERIOD:
 
                kwargs['period'] = Period.THRU_MIDDLE
 
            else:
 
                kwargs['period'] = Period.OPENING
 
        equity_totals = [
 
            -self.balances.total(account=EQUITY_ACCOUNTS, **kwargs)
 
            for kwargs in bal_kwargs
 
        ]
 
        self.write_totals_row("Beginning Net Assets", equity_totals)
 
        self.write_totals_row(
 
            "Ending Net Assets",
 
            income_totals, other_totals, equity_totals,
 
            stylename=self.style_bottomline,
 
        )
 

	
 
    def write_functional_expenses(self) -> None:
 
        self.start_sheet(
 
            "Functional Expenses",
 
            ["Program", "Services"],
 
            ["Management and", "Administrative"],
 
            ["Fundraising"],
 
            totals_prefix=[f"Total {self.period_desc} Ended"],
 
        )
 
        totals = self.write_classifications_by_account('Expenses', [
 
            {'period': Period.PERIOD, 'post_type': 'program'},
 
            {'period': Period.PERIOD, 'post_type': 'management'},
 
            {'period': Period.PERIOD, 'post_type': 'fundraising'},
 
            {'period': Period.PERIOD},
 
            {'period': Period.PRIOR},
 
        ])
 
        self.write_totals_row(
 
            "Total Expenses",
 
            totals,
 
            stylename=self.style_bottomline,
 
        )
 

	
 
    def write_cash_flows(self) -> None:
 
        self.start_sheet("Cash Flows")
 
        bal_kwargs: Sequence[Dict[str, Any]] = [
 
            {'period': Period.PERIOD},
 
            {'period': Period.PRIOR},
 
        ]
 
        norm_func = operator.neg
 

	
 
        self.add_row(self.string_cell(
 
            "Cash Flows from Operating Activities",
 
            stylename=self.style_bold,
 
        ))
 
        equity_totals = [
 
            -self.balances.total(account=EQUITY_ACCOUNTS, **kwargs)
 
            for kwargs in bal_kwargs
 
        ]
 
        self.write_totals_row("Change in Net Assets", equity_totals, leading_rows=1)
 
        self.add_row(self.string_cell(
 
            "(Increase) decrease in operating assets:",
 
        ))
 
        asset_totals = self.write_classifications_by_account(
 
            'Assets', bal_kwargs, (self.C_CASH,), self.SPACE, norm_func,
 
        )
 
        self.add_row(self.string_cell(
 
            "Increase (decrease) in operating liabilities:",
 
        ))
 
        liabilities = self.write_classifications_by_account(
 
            'Liabilities', bal_kwargs, (), self.SPACE, norm_func,
 
        )
 
        period_totals = [
 
            sum(bals, core.MutableBalance())
 
            for bals in zip(equity_totals, asset_totals, liabilities)
 
        ]
 
        self.write_totals_row(
 
            "Net cash provided by operating activities",
 
            period_totals,
 
            stylename=self.style_endtotal,
 
        )
 
        self.write_totals_row("Net Increase in Cash", period_totals)
 
        begin_totals = [
 
            self.balances.total(classification=self.C_CASH, period=period)
 
            for period in [Period.THRU_MIDDLE, Period.OPENING]
 
        ]
 
        self.write_totals_row("Beginning Cash", begin_totals)
 
        self.write_totals_row(
 
            "Ending Cash",
 
            period_totals, begin_totals,
 
            stylename=self.style_bottomline,
 
        )
 

	
 
    def write_trial_balances(self) -> None:
 
        self.start_sheet(
 
            "Trial Balances",
 
            ["Account Name"],
 
            ["Classification"],
 
            ["Balance Beginning", self.balances.period_range.start.strftime(self.date_fmt)],
 
            totals_prefix=["Change During", f"{self.period_desc} Ending"],
 
            title_fmt="Chart of Accounts with DRAFT {sheet_name}",
 
        )
 
        # Widen text columns
 
        col_style = self.column_style(3.5)
 
        for col in self.sheet.childNodes[:2]:
 
            col.setAttribute('stylename', col_style)
 
        # Patch up header row text
 
        header_row = self.sheet.lastChild
 
        header_row.removeChild(header_row.firstChild)
 
        header_row.insertBefore(self.multiline_cell(
 
            ["Balance Ending", self.period_name],
 
            stylename=header_row.lastChild.getAttribute('stylename'),
 
        ), header_row.lastChild)
 

	
 
        for acct_root in ['Assets', 'Liabilities', 'Income', 'Expenses', 'Equity']:
 
            norm_func = core.normalize_amount_func(f'{acct_root}:Dummy')
 
            want_balance = acct_root not in EQUITY_ACCOUNTS
 
            self.add_row()
 
            for account in self.balances.iter_accounts(acct_root):
 
                period_bal = self.balances.total(account=account, period=Period.PERIOD)
 
                prior_bal = self.balances.total(account=account, period=Period.PRIOR)
 
                if want_balance:
 
                    close_bal = self.balances.total(account=account)
 
                    close_cell = self.balance_cell(norm_func(close_bal))
 
                    open_cell = self.balance_cell(norm_func(close_bal - period_bal))
 
                else:
 
                    close_cell = odf.table.TableCell()
 
                    open_cell = odf.table.TableCell()
 
                self.add_row(
 
                    self.string_cell(account),
 
                    self.string_cell(account.meta.get('classification', '')),
 
                    open_cell,
 
                    self.balance_cell(norm_func(period_bal)),
 
                    close_cell,
 
                    self.balance_cell(norm_func(prior_bal)),
 
                )
 

	
 

	
 
def parse_arguments(arglist: Optional[Sequence[str]]=None) -> argparse.Namespace:
 
    parser = argparse.ArgumentParser(prog=PROGNAME)
 
    cliutil.add_version_argument(parser)
 
    parser.add_argument(
 
        '--begin', '--start', '-b',
 
        dest='start_date',
 
        metavar='DATE',
 
        type=cliutil.date_arg,
 
        help="""Date to start reporting entries, inclusive, in YYYY-MM-DD format.
 
The default is one year ago.
 
""")
 
    parser.add_argument(
 
        '--end', '--stop', '-e',
 
        dest='stop_date',
 
        metavar='DATE',
 
        type=cliutil.date_arg,
 
        help="""Date to stop reporting entries, exclusive, in YYYY-MM-DD format.
 
The default is a year after the start date.
 
""")
 
    parser.add_argument(
 
        '--rewrite-rules', '--rewrites', '-r',
 
        action='append',
 
        default=[],
 
        metavar='PATH',
 
        type=Path,
 
        help="""Use rewrite rules from the given YAML file. You can specify
 
this option multiple times to load multiple sets of rewrite rules in order.
 
""")
 
    cliutil.add_rewrite_rules_argument(parser)
 
    parser.add_argument(
 
        '--fund-metadata-key', '-m',
 
        metavar='KEY',
 
        default='project',
 
        help="""Name of the fund metadata key. Default %(default)s.
 
""")
 
    parser.add_argument(
 
        '--unrestricted-fund', '-u',
 
        metavar='PROJECT',
 
        default='Conservancy',
 
        help="""Name of the unrestricted fund. Default %(default)s.
 
""")
 
    parser.add_argument(
 
        '--output-file', '-O',
 
        metavar='PATH',
 
        type=Path,
 
        help="""Write the report to this file, or stdout when PATH is `-`.
 
""")
 
    cliutil.add_loglevel_argument(parser)
 
    return parser.parse_args(arglist)
 

	
 
def main(arglist: Optional[Sequence[str]]=None,
 
         stdout: TextIO=sys.stdout,
 
         stderr: TextIO=sys.stderr,
 
         config: Optional[configmod.Config]=None,
 
) -> int:
 
    args = parse_arguments(arglist)
 
    cliutil.set_loglevel(logger, args.loglevel)
 
    if config is None:
 
        config = configmod.Config()
 
        config.load_file()
 

	
 
    if args.stop_date is None:
 
        if args.start_date is None:
 
            args.stop_date = datetime.date.today()
 
        else:
 
            args.stop_date = cliutil.diff_year(args.start_date, 1)
 
    if args.start_date is None:
 
        args.start_date = cliutil.diff_year(args.stop_date, -1)
 

	
 
    returncode = 0
 
    books_loader = config.books_loader()
 
    if books_loader is None:
 
        entries, load_errors, options_map = books.Loader.load_none(config.config_file_path())
 
        returncode = cliutil.ExitCode.NoConfiguration
 
    else:
 
        start_fy = config.fiscal_year_begin().for_date(args.start_date) - 1
 
        entries, load_errors, options_map = books_loader.load_fy_range(start_fy, args.stop_date)
 
        if load_errors:
 
            returncode = cliutil.ExitCode.BeancountErrors
 
        elif not entries:
 
            returncode = cliutil.ExitCode.NoDataLoaded
 
    for error in load_errors:
 
        bc_printer.print_error(error, file=stderr)
 

	
 
    data.Account.load_from_books(entries, options_map)
 
    postings = data.Posting.from_entries(entries)
 
    for rewrite_path in args.rewrite_rules:
 
        try:
 
            ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path)
 
        except ValueError as error:
 
            logger.critical("failed loading rewrite rules from %s: %s",
 
                            rewrite_path, error.args[0])
 
            return os.EX_DATAERR
 
            return cliutil.ExitCode.RewriteRulesError
 
        postings = ruleset.rewrite(postings)
 

	
 
    balances = Balances(
 
        postings,
 
        args.start_date,
 
        args.stop_date,
 
        args.fund_metadata_key,
 
        args.unrestricted_fund,
 
    )
 
    report = Report(balances)
 
    report.set_common_properties(config.books_repo())
 
    report.write_all()
 
    if args.output_file is None:
 
        out_dir_path = config.repository_path() or Path()
 
        args.output_file = out_dir_path / 'BalanceSheet_{}_{}.ods'.format(
 
            args.start_date.isoformat(), args.stop_date.isoformat(),
 
        )
 
        logger.info("Writing report to %s", args.output_file)
 
    ods_file = cliutil.bytes_output(args.output_file, stdout)
 
    report.save_file(ods_file)
 
    return returncode
 

	
 
entry_point = cliutil.make_entry_point(__name__, PROGNAME)
 

	
 
if __name__ == '__main__':
 
    exit(entry_point())
conservancy_beancount/reports/fund.py
Show inline comments
 
"""funds.py - Funds report from Beancount
 

	
 
This tool produces a report of account balances for restricted funds,
 
including income, expenses, and realized gain/loss over a given time period.
 
Restricted funds are designated with the "project" metadata.
 

	
 
Specify the date range you want to report with the ``--begin`` and ``--end``
 
options.
 

	
 
Select the accounts you want to report with the ``--account`` option. You can
 
specify this option multiple times. The report will include at least one sheet
 
for each account you specify. Subaccounts will be reported on that sheet as
 
well.
 

	
 
Select the postings you want to report by passing metadata search terms in
 
``name=value`` format.
 

	
 
Run ``ledger-report --help`` for abbreviations and other options.
 

	
 
Examples
 
--------
 

	
 
Generate a report of all restricted funds in ODS format over the last year::
 

	
 
      fund-report
 

	
 
Query a specific restricted fund and get a quick balance on the terminal::
 

	
 
      fund-report <PROJECT>
 
"""
 
# Copyright © 2020  Brett Smith
 
#
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU Affero General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU Affero General Public License for more details.
 
#
 
# You should have received a copy of the GNU Affero General Public License
 
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 

	
 
import argparse
 
import collections
 
import datetime
 
import enum
 
import locale
 
import logging
 
import sys
 

	
 
from typing import (
 
    Dict,
 
    Iterable,
 
    Iterator,
 
    Mapping,
 
    Optional,
 
    Sequence,
 
    TextIO,
 
    Tuple,
 
    Union,
 
)
 
from ..beancount_types import (
 
    MetaValue,
 
)
 

	
 
from decimal import Decimal
 
from pathlib import Path
 

	
 
import odf.table  # type:ignore[import]
 

	
 
from beancount.parser import printer as bc_printer
 

	
 
from . import core
 
from . import rewrite
 
from .. import books
 
from .. import cliutil
 
from .. import config as configmod
 
from .. import data
 

	
 
AccountsMap = Mapping[data.Account, core.PeriodPostings]
 
FundPosts = Tuple[MetaValue, AccountsMap]
 

	
 
EQUITY_ACCOUNTS = ['Income', 'Expenses', 'Equity']
 
INFO_ACCOUNTS = [
 
    'Assets:Receivable',
 
    'Assets:Prepaid',
 
    'Liabilities:UnearnedIncome',
 
    'Liabilities:Payable',
 
]
 
PROGNAME = 'fund-report'
 
UNRESTRICTED_FUND = 'Conservancy'
 
logger = logging.getLogger('conservancy_beancount.reports.fund')
 

	
 
class ODSReport(core.BaseODS[FundPosts, None]):
 
    def __init__(self, start_date: datetime.date, stop_date: datetime.date) -> None:
 
        super().__init__()
 
        self.start_date = start_date
 
        self.stop_date = stop_date
 

	
 
    def section_key(self, row: FundPosts) -> None:
 
        return None
 

	
 
    def start_spreadsheet(self, *, expanded: bool=True) -> None:
 
        headers = [["Fund"], ["Balance as of", self.start_date.isoformat()]]
 
        if expanded:
 
            sheet_name = "With Breakdowns"
 
            headers.extend([acct] for acct in EQUITY_ACCOUNTS)
 
        else:
 
            sheet_name = "Fund Report"
 
            headers += [["Additions"], ["Releases from", "Restrictions"]]
 
        headers.append(["Balance as of", self.stop_date.isoformat()])
 
        if expanded:
 
            headers += [
 
                ["Of which", "Receivable"],
 
                ["Of which", "Prepaid Expenses"],
 
                ["Of which", "Payable"],
 
                ["Of which", "Unearned Income"],
 
            ]
 

	
 
        self.use_sheet(sheet_name)
 
        for header in headers:
 
            first_line = header[0]
 
            if first_line == 'Fund':
 
                width = 2.0
 
            elif first_line == 'Balance as of':
 
                width = 1.5
 
            elif first_line == 'Of which':
 
                width = 1.3
 
            else:
 
                width = 1.2
 
            col_style = self.column_style(width)
 
            self.sheet.addElement(odf.table.TableColumn(stylename=col_style))
 

	
 
        center_bold = self.merge_styles(self.style_centertext, self.style_bold)
 
        row = self.add_row(*(
 
            self.multiline_cell(header, stylename=center_bold)
 
            for header in headers
 
        ))
 
        row.firstChild.setAttribute(
 
            'stylename', self.merge_styles(self.style_endtext, self.style_bold),
 
        )
 
        self.lock_first_row()
 
        self.lock_first_column()
 
        self.add_row()
 
        self.add_row(self.string_cell(
 
            f"Fund Report From {self.start_date.isoformat()} To {self.stop_date.isoformat()}",
 
            stylename=center_bold,
 
            numbercolumnsspanned=6,
 
        ))
 
        self.add_row()
 

	
 
    def end_spreadsheet(self) -> None:
 
        start_sheet = self.sheet
 
        self.set_open_sheet(self.sheet)
 
        self.start_spreadsheet(expanded=False)
 
        bal_indexes = [0, 1, 2, 4]
 
        totals = [core.MutableBalance() for _ in bal_indexes]
 
        threshold = Decimal('.5')
 
        for fund, source_bals in self.balances.items():
 
            balances = [source_bals[index] for index in bal_indexes]
 
            # Incorporate Equity changes to Release from Restrictions.
 
            # Note that using -= mutates the balance in a way we don't want.
 
            balances[2] = balances[2] - source_bals[3]
 
            if (not all(bal.clean_copy(threshold).le_zero() for bal in balances)
 
                and fund != UNRESTRICTED_FUND):
 
                self.write_balances(fund, balances)
 
                for total, bal in zip(totals, balances):
 
                    total += bal
 
        self.write_balances('', totals, self.style_bottomline)
 
        self.document.spreadsheet.childNodes.reverse()
 
        self.sheet = start_sheet
 

	
 
    def _row_balances(self, accounts_map: AccountsMap) -> Iterator[core.Balance]:
 
        key_order = [core.OPENING_BALANCE_NAME, *EQUITY_ACCOUNTS, core.ENDING_BALANCE_NAME]
 
        balances: Dict[str, core.Balance] = {key: core.MutableBalance() for key in key_order}
 
        for acct_s, balance in core.account_balances(accounts_map, EQUITY_ACCOUNTS):
 
            if acct_s in balances:
 
                balances[acct_s] = balance
 
            else:
 
                acct_root, _, _ = acct_s.partition(':')
 
                balances[acct_root] += balance
 
        for key in key_order:
 
            if key == 'Expenses':
 
                yield balances[key]
 
            else:
 
                yield -balances[key]
 
        for info_key in INFO_ACCOUNTS:
 
            for _, balance in core.account_balances(accounts_map, [info_key]):
 
                pass
 
            yield core.normalize_amount_func(info_key)(balance)
 

	
 
    def write_balances(self,
 
                       fund: str,
 
                       balances: Iterable[core.Balance],
 
                       style: Union[None, str, odf.style.Style]=None,
 
    ) -> odf.table.TableRow:
 
        return self.add_row(
 
            self.string_cell(fund, stylename=self.style_endtext),
 
            *(self.balance_cell(bal, stylename=style) for bal in balances),
 
        )
 

	
 
    def write_row(self, row: FundPosts) -> None:
 
        fund, accounts_map = row
 
        self.balances[fund] = list(self._row_balances(accounts_map))
 
        if fund != UNRESTRICTED_FUND:
 
            self.write_balances(fund, self.balances[fund])
 

	
 
    def write(self, rows: Iterable[FundPosts]) -> None:
 
        self.balances: Dict[str, Sequence[core.Balance]] = collections.OrderedDict()
 
        super().write(rows)
 
        try:
 
            unrestricted = self.balances[UNRESTRICTED_FUND]
 
        except KeyError:
 
            pass
 
        else:
 
            self.add_row()
 
            self.write_balances("Unrestricted", unrestricted)
 

	
 

	
 
class TextReport:
 
    def __init__(self,
 
                 start_date: datetime.date,
 
                 stop_date: datetime.date,
 
                 out_file: TextIO) -> None:
 
        self.start_date = start_date
 
        self.stop_date = stop_date
 
        self.out_file = out_file
 

	
 
    def _account_balances(self,
 
                          fund: str,
 
                          account_map: AccountsMap,
 
    ) -> Iterator[Tuple[str, Sequence[str]]]:
 
        total_fmt = f'{fund} balance as of {{}}'
 
        for acct_s, balance in core.account_balances(account_map, EQUITY_ACCOUNTS):
 
            if acct_s is core.OPENING_BALANCE_NAME:
 
                acct_s = total_fmt.format(self.start_date.isoformat())
 
            elif acct_s is core.ENDING_BALANCE_NAME:
 
                acct_s = total_fmt.format(self.stop_date.isoformat())
 
            if not acct_s.startswith('Expenses:'):
 
                balance = -balance
 
            yield acct_s, balance.format(None, sep='\0').split('\0')
 
        for _, account in core.sort_and_filter_accounts(account_map, INFO_ACCOUNTS):
 
            balance = account_map[account].stop_bal
 
            if not balance.is_zero():
 
                balance = core.normalize_amount_func(account)(balance)
 
                yield account, balance.format(None, sep='\0').split('\0')
 

	
 
    def write(self, rows: Iterable[FundPosts]) -> None:
 
        output = [
 
            line
 
            for fund, account_map in rows
 
            for line in self._account_balances(fund, account_map)
 
        ]
 
        acct_width = max(len(acct_s) for acct_s, _ in output) + 2
 
        bal_width = max(len(s) for _, bal_s in output for s in bal_s)
 
        bal_width = max(bal_width, 8)
 
        line_fmt = f'{{:>{acct_width}}}  {{:>{bal_width}}}'
 
        print(line_fmt.replace('{:>', '{:^').format("ACCOUNT", "BALANCE"),
 
              file=self.out_file)
 
        fund_start = f' balance as of {self.start_date.isoformat()}'
 
        fund_end = f' balance as of {self.stop_date.isoformat()}'
 
        for acct_s, bal_seq in output:
 
            is_end_bal = acct_s.endswith(fund_end)
 
            if is_end_bal or acct_s.endswith(fund_start):
 
                print(line_fmt.format('─' * acct_width, '─' * bal_width),
 
                      file=self.out_file)
 
            bal_iter = iter(bal_seq)
 
            print(line_fmt.format(acct_s, next(bal_iter)), file=self.out_file)
 
            for bal_s in bal_iter:
 
                print(line_fmt.format('', bal_s), file=self.out_file)
 
            if is_end_bal:
 
                print(line_fmt.format('═' * acct_width, '═' * bal_width),
 
                      file=self.out_file)
 

	
 

	
 
class ReportType(enum.Enum):
 
    TEXT = TextReport
 
    ODS = ODSReport
 
    TXT = TEXT
 
    SPREADSHEET = ODS
 

	
 
    @classmethod
 
    def from_arg(cls, s: str) -> 'ReportType':
 
        try:
 
            return cls[s.upper()]
 
        except KeyError:
 
            raise ValueError(f"no report type matches {s!r}") from None
 

	
 

	
 
def parse_arguments(arglist: Optional[Sequence[str]]=None) -> argparse.Namespace:
 
    parser = argparse.ArgumentParser(prog=PROGNAME)
 
    cliutil.add_version_argument(parser)
 
    parser.add_argument(
 
        '--begin', '--start', '-b',
 
        dest='start_date',
 
        metavar='DATE',
 
        type=cliutil.date_arg,
 
        help="""Date to start reporting entries, inclusive, in YYYY-MM-DD format.
 
The default is one year ago.
 
""")
 
    parser.add_argument(
 
        '--end', '--stop', '-e',
 
        dest='stop_date',
 
        metavar='DATE',
 
        type=cliutil.date_arg,
 
        help="""Date to stop reporting entries, exclusive, in YYYY-MM-DD format.
 
The default is a year after the start date.
 
""")
 
    cliutil.add_rewrite_rules_argument(parser)
 
    parser.add_argument(
 
        '--report-type', '-t',
 
        metavar='TYPE',
 
        type=ReportType.from_arg,
 
        help="""Type of report to generate. `text` gives a plain two-column text
 
report listing accounts and balances over the period, and is the default when
 
you search for a specific project/fund. `ods` produces a higher-level
 
spreadsheet, meant to provide an overview of all funds, and is the default when
 
you don't specify a project/fund.
 
""")
 
    parser.add_argument(
 
        '--output-file', '-O',
 
        metavar='PATH',
 
        type=Path,
 
        help="""Write the report to this file, or stdout when PATH is `-`.
 
The default is stdout for text reports, and a generated filename for ODS
 
reports.
 
""")
 
    cliutil.add_loglevel_argument(parser)
 
    parser.add_argument(
 
        'search_terms',
 
        metavar='FILTER',
 
        type=cliutil.SearchTerm.arg_parser('project', 'rt-id'),
 
        nargs=argparse.ZERO_OR_MORE,
 
        help="""Report on postings that match this criteria. The format is
 
NAME=TERM. TERM is a link or word that must exist in a posting's NAME
 
metadata to match. A single ticket number is a shortcut for
 
`rt-id=rt:NUMBER`. Any other word is a shortcut for `project=TERM`.
 
""")
 
    args = parser.parse_args(arglist)
 
    if args.report_type is None:
 
        if any(term.meta_key == 'project' for term in args.search_terms):
 
            args.report_type = ReportType.TEXT
 
        else:
 
            args.report_type = ReportType.ODS
 
    return args
 

	
 
def main(arglist: Optional[Sequence[str]]=None,
 
         stdout: TextIO=sys.stdout,
 
         stderr: TextIO=sys.stderr,
 
         config: Optional[configmod.Config]=None,
 
) -> int:
 
    args = parse_arguments(arglist)
 
    cliutil.set_loglevel(logger, args.loglevel)
 
    if config is None:
 
        config = configmod.Config()
 
        config.load_file()
 

	
 
    if args.stop_date is None:
 
        if args.start_date is None:
 
            args.stop_date = datetime.date.today()
 
        else:
 
            args.stop_date = cliutil.diff_year(args.start_date, 1)
 
    if args.start_date is None:
 
        args.start_date = cliutil.diff_year(args.stop_date, -1)
 

	
 
    returncode = 0
 
    books_loader = config.books_loader()
 
    if books_loader is None:
 
        entries, load_errors, _ = books.Loader.load_none(config.config_file_path())
 
        returncode = cliutil.ExitCode.NoConfiguration
 
    else:
 
        entries, load_errors, _ = books_loader.load_fy_range(args.start_date, args.stop_date)
 
        if load_errors:
 
            returncode = cliutil.ExitCode.BeancountErrors
 
        elif not entries:
 
            returncode = cliutil.ExitCode.NoDataLoaded
 
    for error in load_errors:
 
        bc_printer.print_error(error, file=stderr)
 

	
 
    postings = (
 
    postings = iter(
 
        post
 
        for post in data.Posting.from_entries(entries)
 
        if post.meta.date < args.stop_date
 
    )
 
    for rewrite_path in args.rewrite_rules:
 
        try:
 
            ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path)
 
        except ValueError as error:
 
            logger.critical("failed loading rewrite rules from %s: %s",
 
                            rewrite_path, error.args[0])
 
            return cliutil.ExitCode.RewriteRulesError
 
        postings = ruleset.rewrite(postings)
 
    for search_term in args.search_terms:
 
        postings = search_term.filter_postings(postings)
 
    fund_postings = {
 
        key: related
 
        for key, related in core.RelatedPostings.group_by_meta(postings, 'project')
 
        if isinstance(key, str)
 
    }
 
    period_cls = core.PeriodPostings.with_start_date(args.start_date)
 
    fund_map = collections.OrderedDict(
 
        (fund, dict(period_cls.group_by_account(fund_postings[fund])))
 
        for fund in sorted(fund_postings, key=lambda s: locale.strxfrm(s.casefold()))
 
    )
 
    if not fund_map:
 
        logger.warning("no matching postings found to report")
 
        returncode = returncode or cliutil.ExitCode.NoDataFiltered
 
    elif args.report_type is ReportType.TEXT:
 
        out_file = cliutil.text_output(args.output_file, stdout)
 
        report = TextReport(args.start_date, args.stop_date, out_file)
 
        report.write(fund_map.items())
 
    else:
 
        ods_report = ODSReport(args.start_date, args.stop_date)
 
        ods_report.set_common_properties(config.books_repo())
 
        ods_report.write(fund_map.items())
 
        if args.output_file is None:
 
            out_dir_path = config.repository_path() or Path()
 
            args.output_file = out_dir_path / 'FundReport_{}_{}.ods'.format(
 
                args.start_date.isoformat(), args.stop_date.isoformat(),
 
            )
 
            logger.info("Writing report to %s", args.output_file)
 
        ods_file = cliutil.bytes_output(args.output_file, stdout)
 
        ods_report.save_file(ods_file)
 
    return returncode
 

	
 
entry_point = cliutil.make_entry_point(__name__, PROGNAME)
 

	
 
if __name__ == '__main__':
 
    exit(entry_point())
conservancy_beancount/reports/ledger.py
Show inline comments
 
"""ledger.py - General ledger report from Beancount
 

	
 
This tool produces a spreadsheet that shows postings in Beancount, organized
 
by account.
 

	
 
Specify the date range you want to report with the ``--begin`` and ``--end``
 
options.
 

	
 
Select the accounts you want to report with the ``--account`` option. You can
 
specify this option multiple times. The report will include at least one sheet
 
for each account you specify. Subaccounts will be reported on that sheet as
 
well.
 

	
 
Select the postings you want to report by passing metadata search terms in
 
``name=value`` format.
 

	
 
Run ``ledger-report --help`` for abbreviations and other options.
 

	
 
Examples
 
--------
 

	
 
Report all activity related to a given project::
 

	
 
    ledger-report project=NAME
 

	
 
Get all Assets postings for a given month to help with reconciliation::
 

	
 
    ledger-report -a Assets -b 2018-05-01 -e 2018-06-01
 
"""
 
# Copyright © 2020  Brett Smith
 
#
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU Affero General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU Affero General Public License for more details.
 
#
 
# You should have received a copy of the GNU Affero General Public License
 
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 

	
 
import argparse
 
import collections
 
import datetime
 
import enum
 
import itertools
 
import operator
 
import logging
 
import sys
 

	
 
from typing import (
 
    Any,
 
    Callable,
 
    Dict,
 
    Iterable,
 
    Iterator,
 
    List,
 
    Mapping,
 
    Optional,
 
    Sequence,
 
    Set,
 
    TextIO,
 
    Tuple,
 
    Union,
 
)
 
from ..beancount_types import (
 
    Transaction,
 
)
 

	
 
from pathlib import Path
 

	
 
import odf.table  # type:ignore[import]
 

	
 
from beancount.core import data as bc_data
 
from beancount.parser import printer as bc_printer
 

	
 
from . import core
 
from . import rewrite
 
from .. import books
 
from .. import cliutil
 
from .. import config as configmod
 
from .. import data
 
from .. import ranges
 
from .. import rtutil
 

	
 
PostTally = List[Tuple[int, data.Account]]
 

	
 
PROGNAME = 'ledger-report'
 
logger = logging.getLogger('conservancy_beancount.reports.ledger')
 

	
 
class LedgerODS(core.BaseODS[data.Posting, None]):
 
    CORE_COLUMNS: Sequence[str] = [
 
        'Date',
 
        data.Metadata.human_name('entity'),
 
        'Description',
 
        'Original Amount',
 
        'Booked Amount',
 
    ]
 
    ACCOUNT_COLUMNS: Dict[str, Sequence[str]] = collections.OrderedDict([
 
        ('Income', ['project', 'rt-id', 'receipt', 'income-type', 'memo']),
 
        ('Expenses', ['project', 'rt-id', 'receipt', 'approval', 'expense-type']),
 
        ('Equity', ['project', 'rt-id']),
 
        ('Assets:Receivable', ['project', 'rt-id', 'invoice', 'approval', 'contract', 'purchase-order']),
 
        ('Liabilities:Payable', ['project', 'rt-id', 'invoice', 'approval', 'contract', 'purchase-order']),
 
        ('Assets:PayPal', ['rt-id', 'paypal-id', 'receipt', 'approval']),
 
        ('Assets', ['rt-id', 'receipt', 'approval', 'bank-statement']),
 
        ('Liabilities', ['rt-id', 'receipt', 'approval', 'bank-statement']),
 
    ])
 
    # Excel 2003 was limited to 65,536 rows per worksheet.
 
    # While we can probably count on all our users supporting more modern
 
    # formats (Excel 2007 supports over 1 million rows per worksheet),
 
    # keeping the default limit conservative seems good to avoid running into
 
    # other limits (like the number of hyperlinks per worksheet), plus just
 
    # better for human organization and readability.
 
    SHEET_SIZE = 65500
 

	
 
    def __init__(self,
 
                 start_date: datetime.date,
 
                 stop_date: datetime.date,
 
                 accounts: Optional[Sequence[str]]=None,
 
                 rt_wrapper: Optional[rtutil.RT]=None,
 
                 sheet_size: Optional[int]=None,
 
                 totals_with_entries: Optional[Sequence[str]]=None,
 
                 totals_without_entries: Optional[Sequence[str]]=None,
 
    ) -> None:
 
        if sheet_size is None:
 
            sheet_size = self.SHEET_SIZE
 
        if totals_with_entries is None:
 
            totals_with_entries = [s for s in self.ACCOUNT_COLUMNS if ':' not in s]
 
        if totals_without_entries is None:
 
            totals_without_entries = totals_with_entries
 
        super().__init__(rt_wrapper)
 
        self.date_range = ranges.DateRange(start_date, stop_date)
 
        self.sheet_size = sheet_size
 
        self.totals_with_entries = totals_with_entries
 
        self.totals_without_entries = totals_without_entries
 
        self.report_name = "Ledger"
 

	
 
        if accounts is None:
 
            self.accounts = set(data.Account.iter_accounts())
 
            self.required_sheet_names = list(self.ACCOUNT_COLUMNS)
 
        else:
 
            self.accounts = set()
 
            self.required_sheet_names = []
 
            for acct_spec in accounts:
 
                subaccounts = frozenset(data.Account.iter_accounts_by_hierarchy(acct_spec))
 
                if subaccounts:
 
                    self.accounts.update(subaccounts)
 
                    self._require_sheet(acct_spec)
 
                else:
 
                    account_roots_map = collections.defaultdict(list)
 
                    for account in data.Account.iter_accounts_by_classification(acct_spec):
 
                        self.accounts.add(account)
 
                        account_roots_map[account.root_part()].append(account)
 
                    if not account_roots_map:
 
                        raise ValueError("unknown account name or classification", acct_spec)
 
                    for root_part, accounts in account_roots_map.items():
 
                        start_count = min(account.count_parts() for account in accounts)
 
                        for count in range(start_count, 1, -1):
 
                            target = accounts[0].root_part(count)
 
                            if all(acct.root_part(count) == target for acct in accounts):
 
                                self._require_sheet(target)
 
                                break
 
                        else:
 
                            self._require_sheet(root_part)
 

	
 
    def _require_sheet(self, new_sheet: str) -> None:
 
        for index, sheet in enumerate(self.required_sheet_names):
 
            if new_sheet == sheet:
 
                break
 
            elif new_sheet.startswith(sheet):
 
                self.required_sheet_names.insert(index, new_sheet)
 
                break
 
        else:
 
            self.required_sheet_names.append(new_sheet)
 

	
 
    def init_styles(self) -> None:
 
        super().init_styles()
 
        self.amount_column = self.column_style(1.2)
 
        self.default_column = self.column_style(1.5)
 
        self.column_styles: Mapping[str, Union[str, odf.style.Style]] = {
 
            'Account': self.column_style(2),  # for TransactionODS
 
            'Date': '',
 
            'Description': self.column_style(2),
 
            'Original Amount': self.amount_column,
 
            'Booked Amount': self.amount_column,
 
            data.Metadata.human_name('project'): self.amount_column,
 
            data.Metadata.human_name('rt-id'): self.amount_column,
 
        }
 

	
 
    @classmethod
 
    def _group_tally(
 
            cls,
 
            tally_by_account: PostTally,
 
            key: Callable[[data.Account], Optional[str]],
 
    ) -> Dict[str, PostTally]:
 
        retval: Dict[str, PostTally] = collections.defaultdict(list)
 
        for count, account in tally_by_account:
 
            item_key = key(account)
 
            if item_key is not None:
 
                retval[item_key].append((count, account))
 
        return retval
 

	
 
    @classmethod
 
    def _split_sheet(
 
            cls,
 
            tally_by_account: PostTally,
 
            sheet_size: int,
 
            sheet_name: str,
 
    ) -> Iterator[str]:
 
        total = 0
 
        for index, (count, account) in enumerate(tally_by_account):
 
            total += count
 
            if total > sheet_size:
 
                break
 
        else:
 
            # All the accounts fit in this sheet.
 
            yield sheet_name
 
            return
 
        if index == 0 and len(tally_by_account) == 1:
 
            # With one account, we can't split any further, so warn and stop.
 
            logger.warning(
 
                "%s has %s rows, over size %s",
 
                account, f'{count:,g}', f'{sheet_size:,g}',
 
            )
 
            yield sheet_name
 
            return
 
        group_func = operator.methodcaller('root_part', sheet_name.count(':') + 2)
 
        maybe_split = cls._group_tally(tally_by_account[:index], group_func)
 
        must_split = cls._group_tally(tally_by_account[index:], group_func)
 
        for subkey, must_split_tally in sorted(must_split.items()):
 
            split_names = cls._split_sheet(
 
                maybe_split.get(subkey, []) + must_split_tally, sheet_size, subkey,
 
            )
 
            # We must be willing to split out at least as many sheets as there
 
            # are accounts that didn't fit. Do that first.
 
            yield from itertools.islice(split_names, len(must_split_tally))
 
            # After that, we can be in one of two cases:
 
            # 1. There is no next sheet. All the accounts, including the
 
            #    maybe_splits and must_splits, fit on planned subsheets.
 
            #    Update state to note we don't need a sheet for them anymore.
 
            # 2. The next sheet is named `subkey`, and is planned to include
 
            #    all of our maybe_split accounts. However, we don't need to
 
            #    yield that sheet name, because those accounts already fit in
 
            #    the sheet we're planning, and it would be a needless split.
 
            next_sheet_name = next(split_names, None)
 
            if next_sheet_name is None:
 
                maybe_split.pop(subkey, None)
 
            else:
 
                assert next_sheet_name == subkey
 
                assert not any(split_names)
 
        if maybe_split:
 
            yield sheet_name
 

	
 
    @classmethod
 
    def plan_sheets(
 
            cls,
 
            tally_by_account: Mapping[data.Account, int],
 
            base_sheets: Sequence[str],
 
            sheet_size: int,
 
    ) -> Sequence[str]:
 
        sorted_tally: PostTally = [
 
            (count, account)
 
            for account, count in tally_by_account.items()
 
        ]
 
        sorted_tally.sort()
 
        split_tally = cls._group_tally(
 
            sorted_tally,
 
            operator.methodcaller('is_under', *base_sheets),
 
        )
 
        return [
 
            sheet_name
 
            for key in base_sheets
 
            for sheet_name in cls._split_sheet(split_tally[key], sheet_size, key)
 
        ]
 

	
 
    # The write() override does its own sectioning by account, so this method
 
    # goes unused.
 
    def section_key(self, row: data.Posting) -> None:
 
        return None
 

	
 
    def metadata_columns_for(self, sheet_name: str) -> Sequence[str]:
 
        columns_key = data.Account(sheet_name).is_under(*self.ACCOUNT_COLUMNS)
 
        # columns_key must not be None because ACCOUNT_COLUMNS has an entry
 
        # for all five root accounts.
 
        assert columns_key is not None
 
        return self.ACCOUNT_COLUMNS[columns_key]
 

	
 
    def start_sheet(self, sheet_name: str) -> None:
 
        self.use_sheet(sheet_name.replace(':', ' '))
 
        self.metadata_columns = self.metadata_columns_for(sheet_name)
 
        self.sheet_columns: Sequence[str] = [
 
            *self.CORE_COLUMNS,
 
            *(data.Metadata.human_name(meta_key) for meta_key in self.metadata_columns),
 
        ]
 
        for col_name in self.sheet_columns:
 
            self.sheet.addElement(odf.table.TableColumn(
 
                stylename=self.column_styles.get(col_name, self.default_column),
 
            ))
 
        self.add_row(*(
 
            self.string_cell(col_name, stylename=self.style_bold)
 
            for col_name in self.sheet_columns
 
        ))
 
        self.lock_first_row()
 

	
 
    def _write_total_row(self,
 
                         date: datetime.date,
 
                         description: str,
 
                         balance: core.Balance,
 
                         bal_style: Union[None, str, odf.style.Style]=None,
 
    ) -> None:
 
        cells: List[odf.table.TableCell] = []
 
        for column in self.CORE_COLUMNS:
 
            if column == 'Date':
 
                cell = self.date_cell(date, stylename=self.merge_styles(
 
                    self.style_bold, self.style_date,
 
                ))
 
            elif column == 'Description':
 
                cell = self.string_cell(description, stylename=self.style_bold)
 
            elif column == 'Booked Amount':
 
                cell = self.balance_cell(self.norm_func(balance), stylename=bal_style)
 
            else:
 
                cell = odf.table.TableCell()
 
            cells.append(cell)
 
        self.add_row(*cells)
 

	
 
    def _report_section_balance(self, key: data.Account, date_key: str) -> None:
 
        related = self.account_groups[key]
 
        if date_key == 'start':
 
            if not key.keeps_balance():
 
                return
 
            date = self.date_range.start
 
            balance = related.start_bal
 
            description = "Opening Balance"
 
            stylename: Optional[odf.style.Style] = None
 
        else:
 
            date = self.date_range.stop
 
            stylename = self.style_bottomline
 
            if key.keeps_balance():
 
                balance = related.stop_bal
 
                description = "Ending Balance"
 
            else:
 
                balance = related.period_bal
 
                description = "Period Total"
 
        self._write_total_row(date, description, balance, stylename)
 

	
 
    def write_header(self, key: data.Account) -> None:
 
        self.add_row()
 
        core_count = len(self.CORE_COLUMNS)
 
        try:
 
            classification = key.meta['classification']
 
        except KeyError:
 
            classification_cell = odf.table.TableCell()
 
        else:
 
            classification_cell = self.string_cell(
 
                classification,
 
                numbercolumnsspanned=len(self.sheet_columns) - core_count,
 
            )
 
        self.add_row(
 
            odf.table.TableCell(),
 
            self.string_cell(
 
                key,
 
                stylename=self.style_bold,
 
                numbercolumnsspanned=core_count - 1,
 
            ),
 
            *(odf.table.TableCell() for _ in range(2, core_count)),
 
            classification_cell,
 
        )
 

	
 
    def write_balance_sheet(self) -> None:
 
        self.use_sheet("Balance")
 
        self.sheet.addElement(odf.table.TableColumn(stylename=self.column_style(3)))
 
        self.sheet.addElement(odf.table.TableColumn(stylename=self.column_style(1.5)))
 
        self.add_row(
 
            self.string_cell("Account", stylename=self.style_bold),
 
            self.string_cell("Balance", stylename=self.style_bold),
 
        )
 
        self.lock_first_row()
 
        self.add_row()
 
        self.add_row(self.string_cell(
 
            f"Ledger From {self.date_range.start.isoformat()}"
 
            f" To {self.date_range.stop.isoformat()}",
 
            stylename=self.merge_styles(self.style_centertext, self.style_bold),
 
            numbercolumnsspanned=2,
 
        ))
 
        self.add_row()
 
        for account, balance in core.account_balances(self.account_groups):
 
            if account is core.OPENING_BALANCE_NAME:
 
                text = f"Balance as of {self.date_range.start.isoformat()}"
 
                style = self.merge_styles(self.style_bold, self.style_endtext)
 
            elif account is core.ENDING_BALANCE_NAME:
 
                text = f"Balance as of {self.date_range.stop.isoformat()}"
 
                style = self.merge_styles(
 
                    self.style_bottomline, self.style_bold, self.style_endtext,
 
                )
 
            else:
 
                text = account
 
                style = self.style_endtext
 
            self.add_row(
 
                self.string_cell(text, stylename=style),
 
                self.balance_cell(-balance, stylename=style),
 
            )
 

	
 
    def _account_tally(self, account: data.Account) -> int:
 
        return len(self.account_groups[account])
 

	
 
    def write_entries(self, account: data.Account, rows: Iterable[data.Posting]) -> None:
 
        for row in rows:
 
            if row.cost is None:
 
                amount_cell = odf.table.TableCell()
 
            else:
 
                amount_cell = self.currency_cell(self.norm_func(row.units))
 
            self.add_row(
 
                self.date_cell(row.meta.date),
 
                self.string_cell(row.meta.get('entity') or ''),
 
                self.string_cell(row.meta.txn.narration),
 
                amount_cell,
 
                self.currency_cell(self.norm_func(row.at_cost())),
 
                *(self.meta_links_cell(row.meta.report_links(key))
 
                  if key in data.LINK_METADATA
 
                  else self.string_cell(row.meta.get(key, ''))
 
                  for key in self.metadata_columns),
 
            )
 

	
 
    def write(self, rows: Iterable[data.Posting]) -> None:
 
        related_cls = core.PeriodPostings.with_start_date(self.date_range.start)
 
        self.account_groups = dict(related_cls.group_by_account(
 
            post for post in rows if post.meta.date < self.date_range.stop
 
        ))
 
        for empty_acct in self.accounts.difference(self.account_groups):
 
            self.account_groups[empty_acct] = related_cls()
 
        self.write_balance_sheet()
 
        tally_by_account_iter = (
 
            (account, self._account_tally(account))
 
            for account in self.accounts
 
        )
 
        tally_by_account = {
 
            # 3 for the rows generated by start_section+end_section
 
            account: count + 3
 
            for account, count in tally_by_account_iter
 
        }
 
        sheet_names = self.plan_sheets(
 
            tally_by_account, self.required_sheet_names, self.sheet_size,
 
        )
 
        using_sheet_index = -1
 
        for sheet_index, account in core.sort_and_filter_accounts(
 
                tally_by_account, sheet_names,
 
        ):
 
            if not account.is_open_on_date(self.date_range.start):
 
                continue
 
            while using_sheet_index < sheet_index:
 
                using_sheet_index += 1
 
                self.start_sheet(sheet_names[using_sheet_index])
 
            self.norm_func = core.normalize_amount_func(account)
 
            postings = self.account_groups[account]
 
            if postings:
 
                totals_set = self.totals_with_entries
 
            else:
 
                totals_set = self.totals_without_entries
 
            want_totals = account.is_under(*totals_set) is not None
 
            if postings or want_totals:
 
                self.write_header(account)
 
            if want_totals:
 
                self._report_section_balance(account, 'start')
 
            self.write_entries(account, postings)
 
            if want_totals:
 
                self._report_section_balance(account, 'stop')
 
        for index in range(using_sheet_index + 1, len(sheet_names)):
 
            self.start_sheet(sheet_names[index])
 

	
 

	
 
class TransactionFilter(enum.IntFlag):
 
    ZERO = 1
 
    CREDIT = 2
 
    DEBIT = 4
 
    ALL = ZERO | CREDIT | DEBIT
 

	
 
    @classmethod
 
    def from_arg(cls, s: str) -> 'TransactionFilter':
 
        try:
 
            return cls[s.upper()]
 
        except KeyError:
 
            raise ValueError(f"unknown transaction filter {s!r}")
 

	
 
    @classmethod
 
    def post_flag(cls, post: data.Posting) -> int:
 
        norm_func = core.normalize_amount_func(post.account)
 
        number = norm_func(post.units.number)
 
        if not number:
 
            return cls.ZERO
 
        elif number > 0:
 
            return cls.CREDIT
 
        else:
 
            return cls.DEBIT
 

	
 

	
 
class TransactionODS(LedgerODS):
 
    CORE_COLUMNS: Sequence[str] = [
 
        'Date',
 
        'Description',
 
        'Account',
 
        data.Metadata.human_name('entity'),
 
        'Original Amount',
 
        'Booked Amount',
 
    ]
 
    METADATA_COLUMNS: Sequence[str] = [
 
        'project',
 
        'rt-id',
 
        'receipt',
 
        'check',
 
        'invoice',
 
        'contract',
 
        'approval',
 
        'paypal-id',
 
        'check-number',
 
        'bank-statement',
 
    ]
 

	
 
    def __init__(self,
 
                 start_date: datetime.date,
 
                 stop_date: datetime.date,
 
                 accounts: Optional[Sequence[str]]=None,
 
                 rt_wrapper: Optional[rtutil.RT]=None,
 
                 sheet_size: Optional[int]=None,
 
                 totals_with_entries: Optional[Sequence[str]]=None,
 
                 totals_without_entries: Optional[Sequence[str]]=None,
 
                 txn_filter: int=TransactionFilter.ALL,
 
    ) -> None:
 
        super().__init__(
 
            start_date,
 
            stop_date,
 
            accounts,
 
            rt_wrapper,
 
            sheet_size,
 
            totals_with_entries,
 
            totals_without_entries,
 
        )
 
        self.txn_filter = txn_filter
 
        if self.txn_filter == TransactionFilter.CREDIT:
 
            self.report_name = "Receipts"
 
        elif self.txn_filter == TransactionFilter.DEBIT:
 
            self.report_name = "Disbursements"
 
        else:
 
            self.report_name = "Transactions"
 

	
 
    def _wanted_txns(self, postings: Iterable[data.Posting]) -> Iterator[Transaction]:
 
        last_txn: Optional[Transaction] = None
 
        for post in postings:
 
            txn = post.meta.txn
 
            if (txn is not last_txn
 
                and TransactionFilter.post_flag(post) & self.txn_filter):
 
                yield txn
 
                last_txn = txn
 

	
 
    def metadata_columns_for(self, sheet_name: str) -> Sequence[str]:
 
        return self.METADATA_COLUMNS
 

	
 
    def write_balance_sheet(self) -> None:
 
        return
 

	
 
    def _report_section_balance(self, key: data.Account, date_key: str) -> None:
 
        if self.txn_filter == TransactionFilter.ALL:
 
            super()._report_section_balance(key, date_key)
 
        elif date_key == 'stop':
 
            balance = core.Balance(
 
                post.at_cost()
 
                for txn in self._wanted_txns(self.account_groups[key])
 
                for post in data.Posting.from_txn(txn)
 
                if post.account == key
 
            )
 
            self._write_total_row(self.date_range.stop, "Period Activity", balance)
 

	
 
    def _account_tally(self, account: data.Account) -> int:
 
        return sum(len(txn.postings)
 
                   for txn in self._wanted_txns(self.account_groups[account]))
 

	
 
    def write_entries(self, account: data.Account, rows: Iterable[data.Posting]) -> None:
 
        for txn in self._wanted_txns(rows):
 
            post_list = list(data.Posting.from_txn(txn))
 
            post_list.sort(key=lambda post: (
 
                0 if post.account == account else 1,
 
                -abs(post.at_cost().number),
 
            ))
 
            postings = iter(post_list)
 
            post1 = next(postings)
 
            if post1.cost is None:
 
                amount_cell = odf.table.TableCell()
 
            else:
 
                amount_cell = self.currency_cell(self.norm_func(post1.units))
 
            self.add_row(
 
                self.date_cell(txn.date),
 
                self.string_cell(txn.narration),
 
                self.string_cell(post1.account),
 
                self.string_cell(post1.meta.get('entity') or ''),
 
                amount_cell,
 
                self.currency_cell(self.norm_func(post1.at_cost())),
 
                *(self.meta_links_cell(post1.meta.report_links(key))
 
                  if key in data.LINK_METADATA
 
                  else self.string_cell(post1.meta.get(key, ''))
 
                  for key in self.metadata_columns),
 
            )
 
            for post in postings:
 
                meta_cells: List[odf.table.TableCell] = []
 
                for meta_key in self.metadata_columns:
 
                    try:
 
                        dup = post.meta[meta_key] is txn.meta[meta_key]
 
                    except KeyError:
 
                        dup = False
 
                    if dup:
 
                        meta_cell = odf.table.TableCell()
 
                    elif meta_key in data.LINK_METADATA:
 
                        meta_cell = self.meta_links_cell(post.meta.report_links(meta_key))
 
                    else:
 
                        meta_cell = self.string_cell(post.meta.get(meta_key, ''))
 
                    meta_cells.append(meta_cell)
 
                if post.cost is None:
 
                    amount_cell = odf.table.TableCell()
 
                else:
 
                    amount_cell = self.currency_cell(self.norm_func(post.units))
 
                self.add_row(
 
                    odf.table.TableCell(),
 
                    odf.table.TableCell(),
 
                    self.string_cell(post.account),
 
                    self.string_cell(post.meta.get('entity') or ''),
 
                    amount_cell,
 
                    self.currency_cell(self.norm_func(post.at_cost())),
 
                    *meta_cells,
 
                )
 

	
 

	
 
class CashReportAction(argparse.Action):
 
    def __call__(self,
 
                 parser: argparse.ArgumentParser,
 
                 namespace: argparse.Namespace,
 
                 values: Union[Sequence[Any], str, None]=None,
 
                 option_string: Optional[str]=None,
 
    ) -> None:
 
        namespace.txn_filter = self.const
 
        if namespace.accounts is None:
 
            namespace.accounts = []
 
        namespace.accounts.append('Assets:PayPal')
 
        namespace.accounts.append('Cash')
 
        if namespace.stop_date is None:
 
            namespace.stop_date = datetime.date.today()
 

	
 

	
 
def parse_arguments(arglist: Optional[Sequence[str]]=None) -> argparse.Namespace:
 
    parser = argparse.ArgumentParser(prog=PROGNAME)
 
    cliutil.add_version_argument(parser)
 
    parser.add_argument(
 
        '--disbursements',
 
        action=CashReportAction,
 
        const=TransactionFilter.DEBIT,
 
        nargs=0,
 
        help="""Shortcut to set all the necessary options to generate a cash
 
disbursements report.
 
""")
 
    parser.add_argument(
 
        '--receipts',
 
        action=CashReportAction,
 
        const=TransactionFilter.CREDIT,
 
        nargs=0,
 
        help="""Shortcut to set all the necessary options to generate a cash
 
receipts report.
 
""")
 
    parser.add_argument(
 
        '--begin', '--start', '-b',
 
        dest='start_date',
 
        metavar='DATE',
 
        type=cliutil.date_arg,
 
        help="""Date to start reporting entries, inclusive, in YYYY-MM-DD format.
 
The default is one year ago.
 
""")
 
    parser.add_argument(
 
        '--end', '--stop', '-e',
 
        dest='stop_date',
 
        metavar='DATE',
 
        type=cliutil.date_arg,
 
        help="""Date to stop reporting entries, exclusive, in YYYY-MM-DD format.
 
The default is a year after the start date, or 30 days from today if the start
 
date was also not specified.
 
""")
 
    parser.add_argument(
 
        '--transactions', '-t',
 
        dest='txn_filter',
 
        metavar='TYPE',
 
        type=TransactionFilter.from_arg,
 
        help="""Report whole transactions rather than individual postings.
 
The type argument selects which type of transactions to report. Choices are
 
credit, debit, or all.
 
""")
 
    parser.add_argument(
 
        '--account', '-a',
 
        dest='accounts',
 
        metavar='ACCOUNT',
 
        action='append',
 
        help="""Show this account in the report. You can specify this option
 
multiple times. You can specify a part of the account hierarchy, or an account
 
classification from metadata. If not specified, the default set adapts to your
 
search criteria.
 
""")
 
    cliutil.add_rewrite_rules_argument(parser)
 
    parser.add_argument(
 
        '--show-totals', '-S',
 
        metavar='ACCOUNT',
 
        action='append',
 
        help="""When entries for this account appear in the report, include
 
account balance(s) as well. You can specify this option multiple times. Pass in
 
a part of the account hierarchy. The default is all accounts.
 
""")
 
    parser.add_argument(
 
        '--add-totals', '-T',
 
        metavar='ACCOUNT',
 
        action='append',
 
        help="""When an account could be included in the report but does not
 
have any entries in the date range, include a header and account balance(s) for
 
it. You can specify this option multiple times. Pass in a part of the account
 
hierarchy. The default set adapts to your search criteria.
 
""")
 
    parser.add_argument(
 
        '--sheet-size', '--size',
 
        metavar='SIZE',
 
        type=int,
 
        default=LedgerODS.SHEET_SIZE,
 
        help="""Try to limit sheets to this many rows. The report will
 
automatically create new sheets to make this happen. When that's not possible,
 
it will issue a warning.
 
""")
 
    parser.add_argument(
 
        '--output-file', '-O',
 
        metavar='PATH',
 
        type=Path,
 
        help="""Write the report to this file, or stdout when PATH is `-`.
 
The default is `LedgerReport_<StartDate>_<StopDate>.ods`.
 
""")
 
    cliutil.add_loglevel_argument(parser)
 
    parser.add_argument(
 
        'search_terms',
 
        metavar='FILTER',
 
        type=cliutil.SearchTerm.arg_parser('project', 'rt-id'),
 
        nargs=argparse.ZERO_OR_MORE,
 
        help="""Report on postings that match this criteria. The format is
 
NAME=TERM. TERM is a link or word that must exist in a posting's NAME
 
metadata to match. A single ticket number is a shortcut for
 
`rt-id=rt:NUMBER`. Any other word is a shortcut for `project=TERM`.
 
""")
 
    args = parser.parse_args(arglist)
 
    if args.add_totals is None and args.search_terms:
 
        args.add_totals = []
 
    if args.accounts is None:
 
        if any(term.meta_key == 'project' for term in args.search_terms):
 
            args.accounts = [
 
                'Income',
 
                'Expenses',
 
                'Assets:Receivable',
 
                'Assets:Prepaid',
 
                'Liabilities:UnearnedIncome',
 
                'Liabilities:Payable',
 
            ]
 
        else:
 
            args.accounts = list(LedgerODS.ACCOUNT_COLUMNS)
 
    return args
 

	
 
def main(arglist: Optional[Sequence[str]]=None,
 
         stdout: TextIO=sys.stdout,
 
         stderr: TextIO=sys.stderr,
 
         config: Optional[configmod.Config]=None,
 
) -> int:
 
    args = parse_arguments(arglist)
 
    cliutil.set_loglevel(logger, args.loglevel)
 
    if config is None:
 
        config = configmod.Config()
 
        config.load_file()
 

	
 
    today = datetime.date.today()
 
    if args.start_date is None:
 
        args.start_date = cliutil.diff_year(today, -1)
 
        if args.stop_date is None:
 
            args.stop_date = today + datetime.timedelta(days=30)
 
    elif args.stop_date is None:
 
        args.stop_date = cliutil.diff_year(args.start_date, 1)
 

	
 
    returncode = 0
 
    books_loader = config.books_loader()
 
    if books_loader is None:
 
        entries, load_errors, options = books.Loader.load_none(config.config_file_path())
 
        returncode = cliutil.ExitCode.NoConfiguration
 
    else:
 
        entries, load_errors, options = books_loader.load_fy_range(args.start_date, args.stop_date)
 
        if load_errors:
 
            returncode = cliutil.ExitCode.BeancountErrors
 
        elif not entries:
 
            returncode = cliutil.ExitCode.NoDataLoaded
 
    for error in load_errors:
 
        bc_printer.print_error(error, file=stderr)
 

	
 
    data.Account.load_from_books(entries, options)
 
    postings = data.Posting.from_entries(entries)
 
    for rewrite_path in args.rewrite_rules:
 
        try:
 
            ruleset = rewrite.RewriteRuleset.from_yaml(rewrite_path)
 
        except ValueError as error:
 
            logger.critical("failed loading rewrite rules from %s: %s",
 
                            rewrite_path, error.args[0])
 
            return cliutil.ExitCode.RewriteRulesError
 
        postings = ruleset.rewrite(postings)
 
    for search_term in args.search_terms:
 
        postings = search_term.filter_postings(postings)
 

	
 
    rt_wrapper = config.rt_wrapper()
 
    if rt_wrapper is None:
 
        logger.warning("could not initialize RT client; spreadsheet links will be broken")
 
    try:
 
        if args.txn_filter is None:
 
            report = LedgerODS(
 
                args.start_date,
 
                args.stop_date,
 
                args.accounts,
 
                rt_wrapper,
 
                args.sheet_size,
 
                args.show_totals,
 
                args.add_totals,
 
            )
 
        else:
 
            report = TransactionODS(
 
                args.start_date,
 
                args.stop_date,
 
                args.accounts,
 
                rt_wrapper,
 
                args.sheet_size,
 
                args.show_totals,
 
                args.add_totals,
 
                args.txn_filter,
 
            )
 
    except ValueError as error:
 
        logger.error("%s: %r", *error.args)
 
        return 2
 
    report.set_common_properties(config.books_repo())
 
    report.write(postings)
 
    if not any(report.account_groups.values()):
 
        logger.warning("no matching postings found to report")
 
        returncode = returncode or cliutil.ExitCode.NoDataFiltered
 

	
 
    if args.output_file is None:
 
        out_dir_path = config.repository_path() or Path()
 
        args.output_file = out_dir_path / '{}Report_{}_{}.ods'.format(
 
            report.report_name,
 
            args.start_date.isoformat(),
 
            args.stop_date.isoformat(),
 
        )
 
        logger.info("Writing report to %s", args.output_file)
 
    ods_file = cliutil.bytes_output(args.output_file, stdout)
 
    report.save_file(ods_file)
 
    return returncode
 

	
 
entry_point = cliutil.make_entry_point(__name__, PROGNAME)
 

	
 
if __name__ == '__main__':
 
    exit(entry_point())
conservancy_beancount/tools/audit_report.py
Show inline comments
 
"""audit_report.py - Utility to run all reports for an audit"""
 
# Copyright © 2020  Brett Smith
 
#
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU Affero General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU Affero General Public License for more details.
 
#
 
# You should have received a copy of the GNU Affero General Public License
 
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 

	
 
import argparse
 
import datetime
 
import logging
 
import multiprocessing
 
import os
 
import runpy
 
import sys
 
import tempfile
 

	
 
from pathlib import Path
 

	
 
from typing import (
 
    Callable,
 
    Iterator,
 
    List,
 
    Optional,
 
    Sequence,
 
    Set,
 
    TextIO,
 
    Tuple,
 
)
 
from types import (
 
    ModuleType,
 
)
 

	
 
from . import extract_odf_links
 
from .. import cliutil
 
from .. import config as configmod
 
from ..reports import accrual
 
from ..reports import balance_sheet
 
from ..reports import fund
 
from ..reports import ledger
 

	
 
from beancount.scripts import check as bc_check
 

	
 
ArgList = List[str]
 
ReportFunc = Callable[[ArgList, TextIO, TextIO, configmod.Config], int]
 

	
 
CPU_COUNT = len(os.sched_getaffinity(0))
 
PROGNAME = 'audit-report'
 
logger = logging.getLogger('conservancy_beancount.tools.audit_report')
 

	
 
def jobs_arg(arg: str) -> int:
 
    if arg.endswith('%'):
 
        arg_n = round(CPU_COUNT * 100 / int(arg[:-1]))
 
    else:
 
        arg_n = int(arg)
 
    return max(1, arg_n)
 

	
 
def parse_arguments(arglist: Optional[Sequence[str]]=None) -> argparse.Namespace:
 
    parser = argparse.ArgumentParser(prog=PROGNAME)
 
    cliutil.add_version_argument(parser)
 
    cliutil.add_loglevel_argument(parser)
 
    parser.add_argument(
 
        '--verbose', '-v',
 
        action='store_true',
 
        help="""Display progress information
 
""")
 
    parser.add_argument(
 
        '--jobs', '-j',
 
        metavar='NUM',
 
        type=jobs_arg,
 
        default=CPU_COUNT,
 
        help="""Maximum number of processes to run concurrently.
 
Can specify a positive integer or a percentage of CPU cores. Default all cores.
 
""")
 
    parser.add_argument(
 
        '--output-directory', '-O',
 
        metavar='DIR',
 
        type=Path,
 
        help="""Write all reports to this directory.
 
Default is a newly-created directory under your repository.
 
""")
 
    parser.add_argument(
 
        '--force',
 
        action='store_true',
 
        help="""Run reports even if bean-check reports errors.
 
""")
 
    parser.add_argument(
 
        '--rewrite-rules', '--rewrite', '-r',
 
        metavar='PATH',
 
        action='append',
 
        type=Path,
 
        default=[],
 
        help="""Path to rewrite rules for the balance sheet.
 
Passed to `balance-sheet-report -r`.
 
""")
 
    parser.add_argument(
 
        '--delimiter', '-d',
 
        metavar='TEXT',
 
        default='\\0',
 
        help="""Delimiter for ODF links in the manifest file.
 
Passed to `extract-odf-links --delimiter`. Default `%(default)s`.
 
""")
 
    parser.add_argument(
 
        'audit_year',
 
        metavar='YEAR',
 
        nargs='?',
 
        type=cliutil.year_or_date_arg,
 
        help="""Main fiscal year to generate reports for.
 
Defaults to the last complete fiscal year.
 
""")
 
    parser.add_argument(
 
        'end_date',
 
        metavar='END',
 
        nargs='?',
 
        type=cliutil.date_arg,
 
        help="""End date for reports for the following fiscal year.
 
The default is automatically calculated from today's date.
 
""")
 
    args = parser.parse_args(arglist)
 
    args.arg_error = parser.error
 
    return args
 

	
 
def now_s() -> str:
 
    return datetime.datetime.now().isoformat(sep=' ', timespec='seconds')
 

	
 
def bean_check(books_path: Path) -> int:
 
    sys.argv = ['bean-check', str(books_path)]
 
    logger.debug("running %r", sys.argv)
 
    # bean-check logs timing information to the root logger at INFO level.
 
    # Suppress that.
 
    logging.getLogger().setLevel(logging.WARNING)
 
    return bc_check.main()  # type:ignore[no-any-return]
 

	
 
def main(arglist: Optional[Sequence[str]]=None,
 
         stdout: TextIO=sys.stdout,
 
         stderr: TextIO=sys.stderr,
 
         config: Optional[configmod.Config]=None,
 
) -> int:
 
    args = parse_arguments(arglist)
 
    if config is None:
 
        config = configmod.Config()
 
        config.load_file()
 
    cliutil.set_loglevel(logger, args.loglevel)
 
    if args.verbose:
 
        logger.setLevel(logging.DEBUG)
 
        reports_logger = logging.getLogger('conservancy_beancount.reports')
 
        reports_logger.setLevel(logging.DEBUG)
 
        reports_logger.getChild('rewrite').setLevel(args.loglevel)
 

	
 
    fy = config.fiscal_year_begin()
 
    today = datetime.date.today()
 
    if args.audit_year is None:
 
        args.audit_year = fy.for_date(today) - 1
 
    audit_begin = fy.first_date(args.audit_year)
 
    audit_end = fy.next_fy_date(args.audit_year)
 
    if args.end_date is None:
 
        days_diff = (today - audit_end).days
 
        if days_diff < (28 * 2):
 
            args.end_date = today
 
        elif days_diff >= 365:
 
            args.end_date = fy.next_fy_date(args.audit_year + 1)
 
        else:
 
            end_date = today - datetime.timedelta(days=today.day + 1)
 
            args.end_date = end_date.replace(day=1)
 
    if args.end_date < audit_end:
 
        args.arg_error("end date is within audited fiscal year")
 
    next_year = fy.for_date(args.end_date)
 
    repo_path = config.repository_path()
 

	
 
    if args.output_directory is None:
 
        args.output_directory = Path(tempfile.mkdtemp(
 
            prefix=f'FY{args.audit_year}AuditReports.', dir=repo_path,
 
        ))
 
        logger.info("writing reports to %s", args.output_directory)
 
    else:
 
        args.output_directory.mkdir(exist_ok=True)
 
    output_reports: List[Path] = []
 
    def common_args(out_name: str, year: Optional[int]=None, *arglist: str) -> Iterator[str]:
 
        if year is not None:
 
            out_name = f'FY{year}{out_name}.ods'
 
        if year == args.audit_year:
 
            yield f'--begin={audit_begin.isoformat()}'
 
            yield f'--end={audit_end.isoformat()}'
 
        elif year == next_year:
 
            yield f'--begin={audit_end.isoformat()}'
 
            yield f'--end={args.end_date.isoformat()}'
 
        elif year is not None:
 
            raise ValueError(f"unknown year {year!r}")
 
        out_path = args.output_directory / out_name
 
        output_reports.append(out_path)
 
        for path in args.rewrite_rules:
 
            yield f'--rewrite-rules={path}'
 
        yield f'--output-file={out_path}'
 
        yield from arglist
 
    reports: List[Tuple[ReportFunc, ArgList]] = [
 
        # Reports are sorted roughly in descending order of how long each takes
 
        # to generate.
 
        (ledger.main, list(common_args('GeneralLedger', args.audit_year))),
 
        (ledger.main, list(common_args('GeneralLedger', next_year))),
 
        (ledger.main, list(common_args('Disbursements', args.audit_year, '--disbursements'))),
 
        (ledger.main, list(common_args('Receipts', args.audit_year, '--receipts'))),
 
        (ledger.main, list(common_args('Disbursements', next_year, '--disbursements'))),
 
        (ledger.main, list(common_args('Receipts', next_year, '--receipts'))),
 
        (accrual.main, list(common_args('AgingReport.ods'))),
 
        (balance_sheet.main, list(common_args(
 
            'Summary', args.audit_year,
 
            *(f'--rewrite-rules={path}' for path in args.rewrite_rules),
 
        ))),
 
        (balance_sheet.main, list(common_args('Summary', args.audit_year))),
 
        (fund.main, list(common_args('FundReport', args.audit_year))),
 
        (fund.main, list(common_args('FundReport', next_year))),
 
    ]
 

	
 
    returncode = 0
 
    books = config.books_loader()
 
    if books is None:
 
        logger.critical("no books available to load")
 
        return os.EX_NOINPUT
 

	
 
    with multiprocessing.Pool(args.jobs, maxtasksperchild=1) as pool:
 
        logger.debug("%s: process pool ready with %s workers", now_s(), args.jobs)
 
        fy_paths = books._iter_fy_books(fy.range(args.audit_year - 1, args.end_date))
 
        check_results = pool.imap_unordered(bean_check, fy_paths)
 
        if all(exitcode == 0 for exitcode in check_results):
 
            logger.debug("%s: bean-check passed", now_s())
 
        else:
 
            logger.log(
 
                logging.WARNING if args.force else logging.ERROR,
 
                "%s: bean-check failed",
 
                now_s(),
 
            )
 
            if not args.force:
 
                return os.EX_DATAERR
 

	
 
        report_results = [
 
            pool.apply_async(report_func, (arglist,), {'config': config})
 
            for report_func, arglist in reports
 
        ]
 
        report_errors = [res.get() for res in report_results if res.get() != 0]
 
        if not report_errors:
 
            logger.debug("%s: all reports generated", now_s())
 
        else:
 
            logger.error("%s: %s reports generated errors", now_s(), len(report_errors))
 
            if not args.force:
 
                return max(report_errors)
 

	
 
    missing_reports = frozenset(
 
        path for path in output_reports if not path.exists()
 
    )
 
    if missing_reports:
 
        logger.error("missing expected reports: %s",
 
                     ', '.join(path.name for path in missing_reports))
 
        if not args.force or len(missing_reports) == len(output_reports):
 
            return os.EX_UNAVAILABLE
 

	
 
    arglist = [f'--delimiter={args.delimiter}']
 
    if repo_path is not None:
 
        arglist.append(f'--relative-to={repo_path}')
 
    arglist.extend(str(p) for p in output_reports if p not in missing_reports)
 
    with (args.output_directory / 'MANIFEST.AUTO').open('w') as manifest_file:
 
        returncode = extract_odf_links.main(arglist, manifest_file, stderr)
 
    logger.debug("%s: manifest generated", now_s())
 
    return returncode
 

	
 
entry_point = cliutil.make_entry_point(__name__, PROGNAME)
 

	
 
if __name__ == '__main__':
 
    exit(entry_point())
setup.py
Show inline comments
 
#!/usr/bin/env python3
 

	
 
from setuptools import setup
 

	
 
setup(
 
    name='conservancy_beancount',
 
    description="Plugin, library, and reports for reading Conservancy's books",
 
    version='1.9.0',
 
    version='1.9.1',
 
    author='Software Freedom Conservancy',
 
    author_email='info@sfconservancy.org',
 
    license='GNU AGPLv3+',
 

	
 
    install_requires=[
 
        'babel>=2.6',  # Debian:python3-babel
 
        'beancount>=2.2',  # Debian:beancount
 
        'GitPython>=2.0',  # Debian:python3-git
 
        # 1.4.1 crashes when trying to save some documents.
 
        'odfpy>=1.4.0,!=1.4.1',  # Debian:python3-odf
 
        'PyYAML>=3.0',  # Debian:python3-yaml
 
        'regex',  # Debian:python3-regex
 
        'rt>=2.0',
 
    ],
 
    setup_requires=[
 
        'pytest-mypy',
 
        'pytest-runner',  # Debian:python3-pytest-runner
 
    ],
 
    tests_require=[
 
        'mypy>=0.770',  # Debian:python3-mypy
 
        'pytest',  # Debian:python3-pytest
 
    ],
 

	
 
    packages=[
 
        'conservancy_beancount',
 
        'conservancy_beancount.plugin',
 
        'conservancy_beancount.reports',
 
    ],
 
    entry_points={
 
        'console_scripts': [
 
            'accrual-report = conservancy_beancount.reports.accrual:entry_point',
 
            'assemble-audit-reports = conservancy_beancount.tools.audit_report:entry_point',
 
            'balance-sheet-report = conservancy_beancount.reports.balance_sheet:entry_point',
 
            'extract-odf-links = conservancy_beancount.tools.extract_odf_links:entry_point',
 
            'fund-report = conservancy_beancount.reports.fund:entry_point',
 
            'ledger-report = conservancy_beancount.reports.ledger:entry_point',
 
            'opening-balances = conservancy_beancount.tools.opening_balances:entry_point',
 
        ],
 
    },
 
)
0 comments (0 inline, 0 general)