From 7281cf0f0179368921af3bc2be3c84c536092217 2020-08-31 02:34:32 From: Brett Smith Date: 2020-08-31 02:34:32 Subject: [PATCH] audit_report: New tool. --- diff --git a/conservancy_beancount/tools/audit_report.py b/conservancy_beancount/tools/audit_report.py new file mode 100644 index 0000000000000000000000000000000000000000..78240ddbbd3200d489fc2c360c808733399d1a11 --- /dev/null +++ b/conservancy_beancount/tools/audit_report.py @@ -0,0 +1,271 @@ +"""audit_report.py - Utility to run all reports for an audit""" +# Copyright © 2020 Brett Smith +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import argparse +import datetime +import logging +import multiprocessing +import os +import runpy +import sys +import tempfile + +from pathlib import Path + +from typing import ( + Callable, + Iterator, + List, + Optional, + Sequence, + Set, + TextIO, + Tuple, +) +from types import ( + ModuleType, +) + +from . import extract_odf_links +from .. import cliutil +from .. import config as configmod +from ..reports import accrual +from ..reports import balance_sheet +from ..reports import fund +from ..reports import ledger + +from beancount.scripts import check as bc_check + +ArgList = List[str] +ReportFunc = Callable[[ArgList, TextIO, TextIO, configmod.Config], int] + +CPU_COUNT = len(os.sched_getaffinity(0)) +PROGNAME = 'audit-report' +logger = logging.getLogger('conservancy_beancount.tools.audit_report') + +def jobs_arg(arg: str) -> int: + if arg.endswith('%'): + arg_n = round(CPU_COUNT * 100 / int(arg[:-1])) + else: + arg_n = int(arg) + return max(1, arg_n) + +def parse_arguments(arglist: Optional[Sequence[str]]=None) -> argparse.Namespace: + parser = argparse.ArgumentParser(prog=PROGNAME) + cliutil.add_version_argument(parser) + cliutil.add_loglevel_argument(parser) + parser.add_argument( + '--verbose', '-v', + action='store_true', + help="""Display progress information +""") + parser.add_argument( + '--jobs', '-j', + metavar='NUM', + type=jobs_arg, + default=CPU_COUNT, + help="""Maximum number of processes to run concurrently. +Can specify a positive integer or a percentage of CPU cores. Default all cores. +""") + parser.add_argument( + '--output-directory', '-O', + metavar='DIR', + type=Path, + help="""Write all reports to this directory. +Default is a newly-created directory under your repository. +""") + parser.add_argument( + '--force', + action='store_true', + help="""Run reports even if bean-check reports errors. +""") + parser.add_argument( + '--rewrite-rules', '--rewrite', '-r', + metavar='PATH', + action='append', + type=Path, + default=[], + help="""Path to rewrite rules for the balance sheet. +Passed to `balance-sheet-report -r`. +""") + parser.add_argument( + '--delimiter', '-d', + metavar='TEXT', + default='\\0', + help="""Delimiter for ODF links in the manifest file. +Passed to `extract-odf-links --delimiter`. Default `%(default)s`. +""") + parser.add_argument( + 'audit_year', + metavar='YEAR', + nargs='?', + type=cliutil.year_or_date_arg, + help="""Main fiscal year to generate reports for. +Defaults to the last complete fiscal year. +""") + parser.add_argument( + 'end_date', + metavar='END', + nargs='?', + type=cliutil.date_arg, + help="""End date for reports for the following fiscal year. +The default is automatically calculated from today's date. +""") + args = parser.parse_args(arglist) + args.arg_error = parser.error + return args + +def now_s() -> str: + return datetime.datetime.now().isoformat(sep=' ', timespec='seconds') + +def bean_check(books_path: Path) -> int: + sys.argv = ['bean-check', str(books_path)] + logger.debug("running %r", sys.argv) + # bean-check logs timing information to the root logger at INFO level. + # Suppress that. + logging.getLogger().setLevel(logging.WARNING) + return bc_check.main() # type:ignore[no-any-return] + +def main(arglist: Optional[Sequence[str]]=None, + stdout: TextIO=sys.stdout, + stderr: TextIO=sys.stderr, + config: Optional[configmod.Config]=None, +) -> int: + args = parse_arguments(arglist) + if config is None: + config = configmod.Config() + config.load_file() + cliutil.set_loglevel(logger, args.loglevel) + if args.verbose: + logger.setLevel(logging.DEBUG) + logging.getLogger('conservancy_beancount.reports').setLevel(logging.DEBUG) + + fy = config.fiscal_year_begin() + today = datetime.date.today() + if args.audit_year is None: + args.audit_year = fy.for_date(today) - 1 + audit_begin = fy.first_date(args.audit_year) + audit_end = fy.next_fy_date(args.audit_year) + if args.end_date is None: + days_diff = (today - audit_end).days + if days_diff < (28 * 2): + args.end_date = today + elif days_diff >= 365: + args.end_date = fy.next_fy_date(args.audit_year + 1) + else: + end_date = today - datetime.timedelta(days=today.day + 1) + args.end_date = end_date.replace(day=1) + if args.end_date < audit_end: + args.arg_error("end date is within audited fiscal year") + next_year = fy.for_date(args.end_date) + repo_path = config.repository_path() + + if args.output_directory is None: + args.output_directory = Path(tempfile.mkdtemp( + prefix=f'FY{args.audit_year}AuditReports.', dir=repo_path, + )) + logger.info("writing reports to %s", args.output_directory) + else: + args.output_directory.mkdir(exist_ok=True) + output_reports: List[Path] = [] + def common_args(out_name: str, year: Optional[int]=None, *arglist: str) -> Iterator[str]: + if year is not None: + out_name = f'FY{year}{out_name}.ods' + if year == args.audit_year: + yield f'--begin={audit_begin.isoformat()}' + yield f'--end={audit_end.isoformat()}' + elif year == next_year: + yield f'--begin={audit_end.isoformat()}' + yield f'--end={args.end_date.isoformat()}' + elif year is not None: + raise ValueError(f"unknown year {year!r}") + out_path = args.output_directory / out_name + output_reports.append(out_path) + yield f'--output-file={out_path}' + yield from arglist + reports: List[Tuple[ReportFunc, ArgList]] = [ + # Reports are sorted roughly in descending order of how long each takes + # to generate. + (ledger.main, list(common_args('GeneralLedger', args.audit_year))), + (ledger.main, list(common_args('GeneralLedger', next_year))), + (ledger.main, list(common_args('Disbursements', args.audit_year, '--disbursements'))), + (ledger.main, list(common_args('Receipts', args.audit_year, '--receipts'))), + (ledger.main, list(common_args('Disbursements', next_year, '--disbursements'))), + (ledger.main, list(common_args('Receipts', next_year, '--receipts'))), + (accrual.main, list(common_args('AgingReport.ods'))), + (balance_sheet.main, list(common_args( + 'Summary', args.audit_year, + *(f'--rewrite-rules={path}' for path in args.rewrite_rules), + ))), + (fund.main, list(common_args('FundReport', args.audit_year))), + (fund.main, list(common_args('FundReport', next_year))), + ] + + returncode = 0 + books = config.books_loader() + if books is None: + logger.critical("no books available to load") + return os.EX_NOINPUT + + with multiprocessing.Pool(args.jobs, maxtasksperchild=1) as pool: + logger.debug("%s: process pool ready with %s workers", now_s(), args.jobs) + fy_paths = books._iter_fy_books(fy.range(args.audit_year - 1, args.end_date)) + check_results = pool.imap_unordered(bean_check, fy_paths) + if all(exitcode == 0 for exitcode in check_results): + logger.debug("%s: bean-check passed", now_s()) + else: + logger.log( + logging.WARNING if args.force else logging.ERROR, + "%s: bean-check failed", + now_s(), + ) + if not args.force: + return os.EX_DATAERR + + report_results = [ + pool.apply_async(report_func, (arglist,), {'config': config}) + for report_func, arglist in reports + ] + report_errors = [res.get() for res in report_results if res.get() != 0] + if not report_errors: + logger.debug("%s: all reports generated", now_s()) + else: + logger.error("%s: %s reports generated errors", now_s(), len(report_errors)) + if not args.force: + return max(report_errors) + + missing_reports = frozenset( + path for path in output_reports if not path.exists() + ) + if missing_reports: + logger.error("missing expected reports: %s", + ', '.join(path.name for path in missing_reports)) + if not args.force or len(missing_reports) == len(output_reports): + return os.EX_UNAVAILABLE + + arglist = [f'--delimiter={args.delimiter}'] + if repo_path is not None: + arglist.append(f'--relative-to={repo_path}') + arglist.extend(str(p) for p in output_reports if p not in missing_reports) + with (args.output_directory / 'MANIFEST.AUTO').open('w') as manifest_file: + returncode = extract_odf_links.main(arglist, manifest_file, stderr) + logger.debug("%s: manifest generated", now_s()) + return returncode + +entry_point = cliutil.make_entry_point(__name__, PROGNAME) + +if __name__ == '__main__': + exit(entry_point()) diff --git a/setup.py b/setup.py index 9979dc3c3ab2ff50b4c75dae8fe5e29bb36424e6..1cd8199c2c4b68ddb6bab459ec9339129a9b85a1 100755 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from setuptools import setup setup( name='conservancy_beancount', description="Plugin, library, and reports for reading Conservancy's books", - version='1.8.8', + version='1.9.0', author='Software Freedom Conservancy', author_email='info@sfconservancy.org', license='GNU AGPLv3+', @@ -37,6 +37,7 @@ setup( entry_points={ 'console_scripts': [ 'accrual-report = conservancy_beancount.reports.accrual:entry_point', + 'assemble-audit-reports = conservancy_beancount.tools.audit_report:entry_point', 'balance-sheet-report = conservancy_beancount.reports.balance_sheet:entry_point', 'extract-odf-links = conservancy_beancount.tools.extract_odf_links:entry_point', 'fund-report = conservancy_beancount.reports.fund:entry_point',