Changeset - aa488effb0f5
[Not reviewed]
0 7 1
Brett Smith - 4 years ago 2020-05-16 14:29:06
brettcsmith@brettcsmith.org
books.Loader: New loading strategy based on load_file. RT#11034.

Building a string and loading it means Beancount can never cache any
load. It only caches top-level file loads because options in the
top-level file can change the semantics of included entries.

Instead use load_file as much as possible, and filter entries as
needed.
8 files changed with 62 insertions and 81 deletions:
0 comments (0 inline, 0 general)
conservancy_beancount/books.py
Show inline comments
...
 
@@ -81,83 +81,68 @@ class FiscalYear(NamedTuple):
 
          range(2015)  # Iterate all fiscal years from 2015 to today, inclusive
 

	
 
          range(-1)  # Iterate the previous fiscal year and current fiscal year
 
        """
 
        if not isinstance(from_fy, int):
 
            from_fy = self.for_date(from_fy)
 
        if to_fy is None:
 
            to_fy = self.for_date()
 
        elif not isinstance(to_fy, int):
 
            to_fy = self.for_date(to_fy - datetime.timedelta(days=1))
 
        if from_fy < 1:
 
            from_fy += to_fy
 
        elif from_fy < 1000:
 
            from_fy, to_fy = to_fy, from_fy + to_fy
 
        return range(from_fy, to_fy + 1)
 

	
 

	
 
class Loader:
 
    """Load Beancount books organized by fiscal year"""
 

	
 
    def __init__(self,
 
                 books_root: Path,
 
                 fiscal_year: FiscalYear,
 
    ) -> None:
 
        """Set up a books loader
 

	
 
        Arguments:
 
        * books_root: A Path to a Beancount books checkout.
 
        * fiscal_year: A FiscalYear object, used to determine what books to
 
          load for a given date range.
 
        """
 
        self.books_root = books_root
 
        self.opening_root = books_root / 'books'
 
        self.fiscal_year = fiscal_year
 

	
 
    def _iter_fy_books(self, fy_range: Iterable[int]) -> Iterator[Path]:
 
        dir_path = self.opening_root
 
        for year in fy_range:
 
            path = dir_path / f'{year}.beancount'
 
            path = Path(self.books_root, 'books', f'{year}.beancount')
 
            if path.exists():
 
                yield path
 
                dir_path = self.books_root
 

	
 
    def fy_range_string(self,
 
                        from_fy: Year,
 
                        to_fy: Optional[Year]=None,
 
    ) -> str:
 
        """Return a string to load books for a range of fiscal years
 
    def load_fy_range(self,
 
                      from_fy: Year,
 
                      to_fy: Optional[Year]=None,
 
    ) -> LoadResult:
 
        """Load books for a range of fiscal years
 

	
 
        This method generates a range of fiscal years by calling
 
        FiscalYear.range() with its first two arguments. It returns a string of
 
        Beancount directives to load the books from the first available fiscal
 
        year through the end of the range.
 

	
 
        Pass the string to Loader.load_string() to actually load data from it.
 
        """
 
        paths = self._iter_fy_books(self.fiscal_year.range(from_fy, to_fy))
 
        fy_range = self.fiscal_year.range(from_fy, to_fy)
 
        fy_paths = self._iter_fy_books(fy_range)
 
        try:
 
            with next(paths).open() as opening_books:
 
                lines = [opening_books.read()]
 
            entries, errors, options_map = bc_loader.load_file(next(fy_paths))
 
        except StopIteration:
 
            return ''
 
        for path in paths:
 
            lines.append(f'include "../{path.name}"')
 
        return '\n'.join(lines)
 

	
 
    def load_string(self, source: str) -> LoadResult:
 
        """Load a generated string of Beancount directives
 

	
 
        This method takes a string generated by another Loader method, like
 
        fy_range_string, and loads it through Beancount, setting up the
 
        environment as necessary to do that.
 
        """
 
        with workdir(self.opening_root):
 
            retval: LoadResult = bc_loader.load_string(source)
 
        return retval
 

	
 
    def load_fy_range(self,
 
                      from_fy: Year,
 
                      to_fy: Optional[Year]=None,
 
    ) -> LoadResult:
 
        """Load books for a range of fiscal years"""
 
        return self.load_string(self.fy_range_string(from_fy, to_fy))
 
            entries, errors, options_map = [], [], {}
 
        for load_path in fy_paths:
 
            new_entries, new_errors, new_options = bc_loader.load_file(load_path)
 
            # We only want transactions from the new fiscal year.
 
            # We don't want the opening balance, duplicate definitions, etc.
 
            fy_filename = str(load_path.parent.parent / load_path.name)
 
            entries.extend(
 
                entry for entry in new_entries
 
                if entry.meta.get('filename') == fy_filename
 
            )
 
            errors.extend(new_errors)
 
        return entries, errors, options_map
tests/books/books/2018.beancount
Show inline comments
 
option "title" "Books from 2018"
 
plugin "beancount.plugins.auto"
 
include "../definitions.beancount"
 
include "../2018.beancount"
tests/books/books/2019.beancount
Show inline comments
 
option "title" "Books from 2019"
 
plugin "beancount.plugins.auto"
 
include "../definitions.beancount"
 
include "../2019.beancount"
tests/books/books/2020.beancount
Show inline comments
 
option "title" "Books from 2020"
 
plugin "beancount.plugins.auto"
 
include "../definitions.beancount"
 
include "../2020.beancount"
tests/books/definitions.beancount
Show inline comments
 
new file 100644
 
2018-03-01 open Assets:Checking
 
2018-03-01 open Income:Donations
tests/test_books_loader.py
Show inline comments
...
 
@@ -3,77 +3,71 @@
 
#
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU Affero General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU Affero General Public License for more details.
 
#
 
# You should have received a copy of the GNU Affero General Public License
 
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 

	
 
import hashlib
 
import re
 

	
 
from datetime import date
 
from pathlib import Path
 

	
 
import pytest
 

	
 
from . import testutil
 

	
 
from conservancy_beancount import books
 

	
 
books_path = testutil.test_path('books')
 

	
 
@pytest.fixture(scope='module')
 
def conservancy_loader():
 
    return books.Loader(books_path, books.FiscalYear(3))
 

	
 
def include_patterns(years, subdir='..'):
 
    for year in years:
 
        path = Path(subdir, f'{year}.beancount')
 
        yield rf'^include "{re.escape(str(path))}"$'
 

	
 
@pytest.mark.parametrize('range_start,range_stop,expect_years', [
 
    (2019, 2020, [2019, 2020]),
 
    (-1, 2020, [2019, 2020]),
 
    (10, 2019, [2019, 2020]),
 
    (-10, 2019, [2018, 2019]),
 
    (date(2019, 1, 1), date(2020, 6, 1), [2018, 2019, 2020]),
 
    (-1, date(2020, 2, 1), [2018, 2019]),
 
@pytest.mark.parametrize('from_fy,to_fy,expect_years', [
 
    (2019, 2019, range(2019, 2020)),
 
    (0, 2019, range(2019, 2020)),
 
    (2018, 2019, range(2018, 2020)),
 
    (1, 2018, range(2018, 2020)),
 
    (-1, 2019, range(2018, 2020)),
 
    (2019, 2020, range(2019, 2021)),
 
    (1, 2019, range(2019, 2021)),
 
    (-1, 2020, range(2019, 2021)),
 
    (2010, 2030, range(2018, 2021)),
 
    (20, 2010, range(2018, 2021)),
 
    (-20, 2030, range(2018, 2021)),
 
])
 
def test_fy_range_string(conservancy_loader, range_start, range_stop, expect_years):
 
    actual = conservancy_loader.fy_range_string(range_start, range_stop)
 
    testutil.check_lines_match(actual.splitlines(), [
 
        rf'^option "title" "Books from {expect_years[0]}"$',
 
        rf'^plugin "beancount\.plugins\.auto"$',
 
        *include_patterns(expect_years),
 
    ])
 

	
 
@pytest.mark.parametrize('year_offset', range(-3, 1))
 
def test_fy_range_string_with_offset(conservancy_loader, year_offset):
 
    base_year = 2020
 
    start_year = max(2018, base_year + year_offset)
 
    expect_years = range(start_year, base_year + 1)
 
    actual = conservancy_loader.fy_range_string(year_offset, base_year)
 
    testutil.check_lines_match(actual.splitlines(), include_patterns(expect_years))
 

	
 
def test_fy_range_string_empty_range(conservancy_loader):
 
    assert conservancy_loader.fy_range_string(2020, 2019) == ''
 

	
 
def test_load_fy_range(conservancy_loader):
 
    entries, errors, options_map = conservancy_loader.load_fy_range(2018, 2019)
 
def test_load_fy_range(conservancy_loader, from_fy, to_fy, expect_years):
 
    entries, errors, options_map = conservancy_loader.load_fy_range(from_fy, to_fy)
 
    assert not errors
 
    narrations = {getattr(entry, 'narration', None) for entry in entries}
 
    assert '2018 donation' in narrations
 
    assert '2019 donation' in narrations
 
    assert '2020 donation' not in narrations
 
    assert ('2018 donation' in narrations) == (2018 in expect_years)
 
    assert ('2019 donation' in narrations) == (2019 in expect_years)
 
    assert ('2020 donation' in narrations) == (2020 in expect_years)
 

	
 
def test_load_fy_range_does_not_duplicate_openings(conservancy_loader):
 
    entries, errors, options_map = conservancy_loader.load_fy_range(2010, 2030)
 
    openings = []
 
    open_accounts = set()
 
    for entry in entries:
 
        try:
 
            open_accounts.add(entry.account)
 
        except AttributeError:
 
            pass
 
        else:
 
            openings.append(entry)
 
    assert len(openings) == len(open_accounts)
 

	
 
def test_load_fy_range_empty(conservancy_loader):
 
    entries, errors, options_map = conservancy_loader.load_fy_range(2020, 2019)
 
    assert not errors
 
    assert not entries
 
    assert options_map.get('input_hash') == hashlib.md5().hexdigest()
 
    assert not options_map
tests/test_config.py
Show inline comments
...
 
@@ -369,36 +369,38 @@ def test_no_books_path():
 
    (' 10 . 30 ', 10, 30),
 
])
 
def test_fiscal_year_begin(value, month, day):
 
    config = config_mod.Config()
 
    config.load_string(f'[Beancount]\nfiscal year begin = {value}\n')
 
    assert config.fiscal_year_begin() == (month, day)
 

	
 
@pytest.mark.parametrize('value', [
 
    'text',
 
    '1900',
 
    '13',
 
    '010',
 
    '2 30',
 
    '4-31',
 
])
 
def test_bad_fiscal_year_begin(value):
 
    config = config_mod.Config()
 
    config.load_string(f'[Beancount]\nfiscal year begin = {value}\n')
 
    with pytest.raises(ValueError):
 
        config.fiscal_year_begin()
 

	
 
def test_default_fiscal_year_begin():
 
    config = config_mod.Config()
 
    actual = config.fiscal_year_begin()
 
    assert actual.month == 3
 
    assert actual.day == 1
 

	
 
def test_books_loader():
 
    books_path = testutil.test_path('books')
 
    config = config_mod.Config()
 
    config.load_string(f'[Beancount]\nbooks dir = {books_path}\n')
 
    loader = config.books_loader()
 
    assert loader.fy_range_string(2020, 2020)
 
    entries, errors, _ = loader.load_fy_range(2020, 2020)
 
    assert entries
 
    assert not errors
 

	
 
def test_books_loader_without_books():
 
    assert config_mod.Config().books_loader() is None
tests/testutil.py
Show inline comments
...
 
@@ -154,68 +154,66 @@ OPENING_EQUITY_ACCOUNTS = itertools.cycle([
 
    'Equity:Funds:Unrestricted',
 
    'Equity:Funds:Restricted',
 
    'Equity:OpeningBalance',
 
])
 

	
 
def balance_map(source=None, **kwargs):
 
    # The source and/or kwargs should map currency name strings to
 
    # things you can pass to Decimal (a decimal string, an int, etc.)
 
    # This returns a dict that maps currency name strings to Amount instances.
 
    retval = {}
 
    if source is not None:
 
        retval.update((currency, Amount(number, currency))
 
                      for currency, number in source)
 
    if kwargs:
 
        retval.update(balance_map(kwargs.items()))
 
    return retval
 

	
 
def OpeningBalance(acct=None, **txn_meta):
 
    if acct is None:
 
        acct = next(OPENING_EQUITY_ACCOUNTS)
 
    return Transaction(**txn_meta, postings=[
 
        ('Assets:Receivable:Accounts', 100),
 
        ('Assets:Receivable:Loans', 200),
 
        ('Liabilities:Payable:Accounts', -15),
 
        ('Liabilities:Payable:Vacation', -25),
 
        (acct, -260),
 
    ])
 

	
 
class TestBooksLoader(books.Loader):
 
    def __init__(self, source):
 
        self.source = source
 

	
 
    def fy_range_string(self, from_fy=None, to_fy=None, plugins=None):
 
        return f'include "{self.source}"'
 

	
 
    load_string = staticmethod(bc_loader.load_string)
 
    def load_fy_range(self, from_fy, to_fy=None):
 
        return bc_loader.load_file(self.source)
 

	
 

	
 
class TestConfig:
 
    def __init__(self, *,
 
                 books_path=None,
 
                 payment_threshold=0,
 
                 repo_path=None,
 
                 rt_client=None,
 
    ):
 
        if books_path is None:
 
            self._books_loader = None
 
        else:
 
            self._books_loader = TestBooksLoader(books_path)
 
        self._payment_threshold = Decimal(payment_threshold)
 
        self.repo_path = test_path(repo_path)
 
        self._rt_client = rt_client
 
        if rt_client is None:
 
            self._rt_wrapper = None
 
        else:
 
            self._rt_wrapper = rtutil.RT(rt_client)
 

	
 
    def books_loader(self):
 
        return self._books_loader
 

	
 
    def config_file_path(self):
 
        return test_path('userconfig/conservancy_beancount/config.ini')
 

	
 
    def payment_threshold(self):
 
        return self._payment_threshold
 

	
 
    def repository_path(self):
 
        return self.repo_path
0 comments (0 inline, 0 general)