NPO-Accounting/conservancy_beancount Changeset - 072937eff508

Changeset - 072937eff508

Parent rev.

Child rev.

[Not reviewed]

0 5 6

Brett Smith - 4 years ago 2020-05-05 18:31:08
brettcsmith@brettcsmith.org

books.Loader: New loading strategy.

The old loading strategy didn't load options, which yielded some
spurious errors. It also created awkward duplication of plugin
information in the code as well as the books.

Implement a new loading strategy that works by reading one of the
"main files" under the books/ subdirectory and includes entries
for additional FYs beyond that.

This is still not ideal in a lot of ways. In particular, Beancount can't
cache any results, causing any load to be slower than it theoretically could
be. I expect more commits to follow. But some of them might require
restructuring the books, and that should happen separately.

11 files changed with 102 insertions and 89 deletions:

conservancy_beancount/books.py

tests/books/2018.beancount

tests/books/2019.beancount

tests/books/2020.beancount

tests/books/books/2018.beancount

tests/books/books/2019.beancount

tests/books/books/2020.beancount

tests/test_books_loader.py

tests/test_config.py

tests/test_reports_accrual.py

tests/testutil.py

0 comments (0 inline, 0 general)

conservancy_beancount/books.py

➞

Show inline comments

@@ ... / @@ -16,3 +16,5 @@ @@
 import contextlib
 import datetime
 import os
@@ ... / @@ -25,2 +27,3 @@ from typing import ( @@
     Iterable,
     Iterator,
     Mapping,
@@ ... / @@ -35,5 +38,13 @@ from .beancount_types import ( @@
 PathLike = Union[str, Path]
 PluginsSpec = Mapping[str, Optional[str]]
 Year = Union[int, datetime.date]
 @contextlib.contextmanager
 def workdir(path: PathLike) -> Iterator[Path]:
     old_dir = os.getcwd()
     os.chdir(path)
     try:
         yield Path(old_dir)
     finally:
         os.chdir(old_dir)
 class FiscalYear(NamedTuple):
@@ ... / @@ -89,6 +100,2 @@ class Loader: @@
     DEFAULT_PLUGINS: PluginsSpec = {
         'conservancy_beancount.plugin': None,
+    }
     def __init__(self,
@@ ... / @@ -96,3 +103,2 @@ class Loader: @@
                  fiscal_year: FiscalYear,
                  plugins: Optional[PluginsSpec]=None,
     ) -> None:
@@ ... / @@ -104,23 +110,14 @@ class Loader: @@
           load for a given date range.
         * plugins: A mapping that specifies what plugins should be loaded
           before any books. The keys are plugin names, and the values are the
           configuration parameters string to follow. A value of None means the
           plugin takes no configuration string. By default, the loader loads
           conservancy_beancount.plugin.
         """
         if plugins is None:
             plugins = self.DEFAULT_PLUGINS
         self.books_root = books_root
         self.opening_root = books_root / 'books'
         self.fiscal_year = fiscal_year
         self.plugins = dict(plugins)
     def _format_include(self, year: int, subdir: PathLike='') -> str:
         file_path = Path(self.books_root, subdir, f'{year}.beancount')
         return f'include "{file_path}"'
     def _format_plugin(self, name: str, optstring: Optional[str]=None) -> str:
         if optstring is None:
             return f'plugin "{name}"'
         else:
             return f'plugin "{name}" "{optstring}"'
     def _iter_fy_books(self, fy_range: Iterable[int]) -> Iterator[Path]:
         dir_path = self.opening_root
         for year in fy_range:
             path = dir_path / f'{year}.beancount'
             if path.exists():
                 yield path
                 dir_path = self.books_root
@@ ... / @@ -129,3 +126,2 @@ class Loader: @@
                         to_fy: Optional[Year]=None,
                         plugins: Optional[PluginsSpec]=None,
     ) -> str:
@@ ... / @@ -135,22 +131,27 @@ class Loader: @@
         FiscalYear.range() with its first two arguments. It returns a string of
         Beancount directives to load all plugins and Beancount files for that
         range of fiscal years, suitable for passing to
         beancount.loader.load_string().
         Beancount directives to load the books from the first available fiscal
         year through the end of the range.
         You can specify what plugins to load with the plugins argument. If not
         specified, the string loads the plugins specified for this instance.
         See the __init__ docstring for details.
         Pass the string to Loader.load_string() to actually load data from it.
         """
         if plugins is None:
             plugins = self.plugins
         years = iter(self.fiscal_year.range(from_fy, to_fy))
         paths = self._iter_fy_books(self.fiscal_year.range(from_fy, to_fy))
         try:
             books_start = self._format_include(next(years), 'books')
             with next(paths).open() as opening_books:
                 lines = [opening_books.read()]
         except StopIteration:
             return ''
         return '\n'.join([
             *(self._format_plugin(name, opts) for name, opts in plugins.items()),
             books_start,
             *(self._format_include(year) for year in years),
         ])
         for path in paths:
             lines.append(f'include "../{path.name}"')
         return '\n'.join(lines)
     def load_string(self, source: str) -> LoadResult:
         """Load a generated string of Beancount directives
         This method takes a string generated by another Loader method, like
         fy_range_string, and loads it through Beancount, setting up the
         environment as necessary to do that.
         """
         with workdir(self.opening_root):
             retval: LoadResult = bc_loader.load_string(source)
         return retval
@@ ... / @@ -159,12 +160,4 @@ class Loader: @@
                       to_fy: Optional[Year]=None,
                       plugins: Optional[PluginsSpec]=None,
     ) -> LoadResult:
         """Load books for a range of fiscal years
         This is a convenience wrapper to call
         self.fy_range_string(from_fy, to_fy, plugins)
         and load the result with beancount.loader.load_string.
         """
         return bc_loader.load_string(  # type:ignore[no-any-return]
             self.fy_range_string(from_fy, to_fy, plugins),
+        )
         """Load books for a range of fiscal years"""
         return self.load_string(self.fy_range_string(from_fy, to_fy))

tests/books/2018.beancount

➞

Show inline comments

 new file 100644
 -04-01 * "2018 donation"
   Income:Donations  20.18 USD
   Assets:Checking

tests/books/2019.beancount

➞

Show inline comments

 new file 100644
 -04-01 * "2019 donation"
   Income:Donations  20.19 USD
   Assets:Checking

tests/books/2020.beancount

➞

Show inline comments

 new file 100644
 -04-01 * "2020 donation"
   Income:Donations  20.20 USD
   Assets:Checking

tests/books/books/2018.beancount

➞

Show inline comments

 new file 100644
 option "title" "Books from 2018"
 plugin "beancount.plugins.auto"
 include "../2018.beancount"

tests/books/books/2019.beancount

➞

Show inline comments

 new file 100644
 option "title" "Books from 2019"
 plugin "beancount.plugins.auto"
 include "../2019.beancount"

tests/books/books/2020.beancount

➞

Show inline comments

 new file 100644
 option "title" "Books from 2020"
 plugin "beancount.plugins.auto"
 include "../2020.beancount"

tests/test_books_loader.py

➞

Show inline comments

@@ ... / @@ -17,2 +17,3 @@ @@
 import hashlib
 import re
@@ ... / @@ -27,3 +28,3 @@ from conservancy_beancount import books @@
-books_path = testutil.test_path('booksroot')
+books_path = testutil.test_path('books')
@@ ... / @@ -33,20 +34,6 @@ def conservancy_loader(): @@
 def format_include(year, subdir=''):
     path = Path(books_path, subdir, f'{year}.beancount')
     return f'include "{path}"'
 def format_plugin(name, optstring=None):
     if optstring is None:
         return f'plugin "{name}"'
     else:
         return f'plugin "{name}" "{optstring}"'
 def expect_string(years, plugins={'conservancy_beancount.plugin': None}):
     years = iter(years)
     year1_s = format_include(next(years), 'books')
     return '\n'.join([
         *(format_plugin(name, opts) for name, opts in plugins.items()),
         year1_s,
         *(format_include(year) for year in years),
     ])
 def include_patterns(years, subdir='..'):
     for year in years:
         path = Path(subdir, f'{year}.beancount')
         yield rf'^include "{re.escape(str(path))}"$'
@@ ... @@
     (-1, 2020, [2019, 2020]),
     (date(2019, 1, 1), date(2020, 6, 1), range(2018, 2021)),
     (10, 2019, [2019, 2020]),
     (-10, 2019, [2018, 2019]),
     (date(2019, 1, 1), date(2020, 6, 1), [2018, 2019, 2020]),
     (-1, date(2020, 2, 1), [2018, 2019]),
@@ ... @@
 def test_fy_range_string(conservancy_loader, range_start, range_stop, expect_years):
     expected = expect_string(expect_years)
     assert conservancy_loader.fy_range_string(range_start, range_stop) == expected
     actual = conservancy_loader.fy_range_string(range_start, range_stop)
     testutil.check_lines_match(actual.splitlines(), [
         rf'^option "title" "Books from {expect_years[0]}"$',
         rf'^plugin "beancount\.plugins\.auto"$',
         *include_patterns(expect_years),
     ])
 @pytest.mark.parametrize('year_offset', range(-3, 1))
 def test_fy_range_string_one_offset(conservancy_loader, year_offset):
     this_year = date.today().year
     expected = expect_string(range(this_year + year_offset, this_year + 1))
     assert conservancy_loader.fy_range_string(year_offset) == expected
 @pytest.mark.parametrize('plugins', [
     {},
     {'conservancy_beancount.plugin': '-all'},
 ])
 def test_fy_range_string_plugins_override(conservancy_loader, plugins):
     expected = expect_string([2019, 2020], plugins)
     assert conservancy_loader.fy_range_string(2019, 2020, plugins) == expected
 def test_fy_range_string_with_offset(conservancy_loader, year_offset):
     base_year = 2020
     start_year = max(2018, base_year + year_offset)
     expect_years = range(start_year, base_year + 1)
     actual = conservancy_loader.fy_range_string(year_offset, base_year)
     testutil.check_lines_match(actual.splitlines(), include_patterns(expect_years))
@@ ... / @@ -79,4 +66,13 @@ def test_fy_range_string_empty_range(conservancy_loader): @@
 def test_load_fy_range(conservancy_loader):
     entries, errors, options_map = conservancy_loader.load_fy_range(2018, 2019)
     assert not errors
     narrations = {getattr(entry, 'narration', None) for entry in entries}
     assert '2018 donation' in narrations
     assert '2019 donation' in narrations
     assert '2020 donation' not in narrations
 def test_load_fy_range_empty(conservancy_loader):
     entries, errors, options_map = conservancy_loader.load_fy_range(2020, 2019)
     assert not errors
     assert not entries

tests/test_config.py

➞

Show inline comments

@@ ... / @@ -385,3 +385,3 @@ def test_default_fiscal_year_begin(): @@
 def test_books_loader():
-    books_path = testutil.test_path('bookstest')
+    books_path = testutil.test_path('books')
     config = config_mod.Config()
@@ ... / @@ -389,4 +389,3 @@ def test_books_loader(): @@
     loader = config.books_loader()
     expected = 'include "{}"'.format(books_path / 'books/2020.beancount')
     assert loader.fy_range_string(0, 2020, {}) == expected
     assert loader.fy_range_string(2020, 2020)

tests/test_reports_accrual.py

➞

Show inline comments

@@ ... / @@ -275,5 +275,3 @@ def check_output(output, expect_patterns): @@
     output.seek(0)
     for pattern in expect_patterns:
         assert any(re.search(pattern, line) for line in output), \
             f"{pattern!r} not found in output"
     testutil.check_lines_match(iter(output), expect_patterns)

tests/testutil.py

➞

Show inline comments

@@ ... / @@ -22,2 +22,3 @@ import beancount.core.amount as bc_amount @@
 import beancount.core.data as bc_data
 import beancount.loader as bc_loader
@@ ... / @@ -35,2 +36,7 @@ TESTS_DIR = Path(__file__).parent @@
 def check_lines_match(lines, expect_patterns, source='output'):
     for pattern in expect_patterns:
         assert any(re.search(pattern, line) for line in lines), \
             f"{pattern!r} not found in {source}"
 def check_post_meta(txn, *expected_meta, default=None):
@@ ... / @@ -182,2 +188,4 @@ class TestBooksLoader(books.Loader): @@
     load_string = staticmethod(bc_loader.load_string)

0 comments (0 inline, 0 general)