Changeset - 2cb131423f97
[Not reviewed]
0 6 0
Brett Smith - 4 years ago 2020-03-28 13:47:40
brettcsmith@brettcsmith.org
errors: Redo InvalidMetadataError.

This needs to be generally usable for transactions.
6 files changed with 13 insertions and 27 deletions:
0 comments (0 inline, 0 general)
conservancy_beancount/errors.py
Show inline comments
 
"""Error classes for plugins to report problems in the books"""
 
# Copyright © 2020  Brett Smith
 
#
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU Affero General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU Affero General Public License for more details.
 
#
 
# You should have received a copy of the GNU Affero General Public License
 
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 

	
 
from typing import (
 
    Iterable,
 
)
 

	
 
class Error(Exception):
 
    def __init__(self, message, entry, source=None):
 
        self.message = message
 
        self.entry = entry
 
        self.source = entry.meta if source is None else source
 

	
 
    def __repr__(self):
 
        return "{clsname}<{source[filename]}:{source[lineno]}: {message}>".format(
 
            clsname=type(self).__name__,
 
            message=self.message,
 
            source=self.source,
 
        )
 

	
 
    def _fill_source(self, source, filename='conservancy_beancount', lineno=0):
 
        source.setdefault('filename', filename)
 
        source.setdefault('lineno', lineno)
 

	
 

	
 
Iter = Iterable[Error]
 

	
 
class BrokenLinkError(Error):
 
    def __init__(self, txn, key, link, source=None):
 
        super().__init__(
 
            "{} not found in repository: {}".format(key, link),
 
            txn,
 
            source,
 
        )
 

	
 
class BrokenRTLinkError(Error):
 
    def __init__(self, txn, key, link, parsed=True, source=None):
 
        if parsed:
 
            msg_fmt = "{} not found in RT: {}"
 
        else:
 
            msg_fmt = "{} link is malformed: {}"
 
        super().__init__(
 
            msg_fmt.format(key, link),
 
            txn,
 
            source,
 
        )
 

	
 
class ConfigurationError(Error):
 
    def __init__(self, message, entry=None, source=None):
 
        if source is None:
 
            source = {}
 
        self._fill_source(source)
 
        super().__init__(message, entry, source)
 

	
 

	
 
class InvalidMetadataError(Error):
 
    def __init__(self, txn, post, key, value=None, source=None):
 
        if value is None:
 
            msg_fmt = "{post.account} missing {key}"
 
        else:
 
            msg_fmt = "{post.account} has invalid {key}: {value}"
 
        super().__init__(
 
            msg_fmt.format(post=post, key=key, value=value),
 
            txn,
 
            source,
 
        )
 

	
 

	
 
class InvalidEntityError(InvalidMetadataError):
 
    def __init__(self, txn, post=None, key='entity', value=None, source=None):
 
    def __init__(self, txn, key, value=None, post=None, source=None):
 
        if post is None:
 
            srcname = 'transaction'
 
        else:
 
            srcname = post.account
 
        if value is None:
 
            msg = "{} missing entity".format(srcname)
 
            msg = "{} missing {}".format(srcname, key)
 
        else:
 
            msg = "{} entity malformed: {}".format(srcname, value)
 
        super(InvalidMetadataError, self).__init__(msg, txn, source)
 
            msg = "{} has invalid {}: {}".format(srcname, key, value)
 
        super().__init__(msg, txn, source)
conservancy_beancount/plugin/core.py
Show inline comments
...
 
@@ -92,150 +92,150 @@ class _GenericRange(Generic[CT]):
 
    `foo in generic_range` is equivalent to `start <= foo < stop`.
 
    Since we have multiple user-configurable ranges, having the check
 
    encapsulated in an object helps implement the check consistently, and
 
    makes it easier for subclasses to override.
 
    """
 

	
 
    def __init__(self, start: CT, stop: CT) -> None:
 
        self.start = start
 
        self.stop = stop
 

	
 
    def __repr__(self) -> str:
 
        return "{clsname}({self.start!r}, {self.stop!r})".format(
 
            clsname=type(self).__name__,
 
            self=self,
 
        )
 

	
 
    def __contains__(self, item: CT) -> bool:
 
        return self.start <= item < self.stop
 

	
 

	
 
class MetadataEnum:
 
    """Map acceptable metadata values to their normalized forms.
 

	
 
    When a piece of metadata uses a set of allowed values, use this class to
 
    define them. You can also specify aliases that hooks will normalize to
 
    the primary values.
 
    """
 

	
 
    def __init__(self,
 
                 key: MetaKey,
 
                 standard_values: Iterable[MetaValueEnum],
 
                 aliases_map: Optional[Mapping[MetaValueEnum, MetaValueEnum]]=None,
 
    ) -> None:
 
        """Specify allowed values and aliases for this metadata.
 

	
 
        Arguments:
 

	
 
        * key: The name of the metadata key that uses this enum.
 
        * standard_values: A sequence of strings that enumerate the standard
 
          values for this metadata.
 
        * aliases_map: A mapping of strings to strings. The keys are
 
          additional allowed metadata values. The values are standard values
 
          that each key will evaluate to. The code asserts that all values are
 
          in standard_values.
 
        """
 
        self.key = key
 
        self._stdvalues = frozenset(standard_values)
 
        self._aliases: Dict[MetaValueEnum, MetaValueEnum] = dict(aliases_map or ())
 
        assert self._stdvalues.issuperset(self._aliases.values())
 
        self._aliases.update((v, v) for v in standard_values)
 

	
 
    def __repr__(self) -> str:
 
        return "{}<{}>".format(type(self).__name__, self.key)
 

	
 
    def __contains__(self, key: MetaValueEnum) -> bool:
 
        """Returns true if `key` is a standard value or alias."""
 
        return key in self._aliases
 

	
 
    def __getitem__(self, key: MetaValueEnum) -> MetaValueEnum:
 
        """Return the standard value for `key`.
 

	
 
        Raises KeyError if `key` is not a known value or alias.
 
        """
 
        return self._aliases[key]
 

	
 
    def __iter__(self) -> Iterator[MetaValueEnum]:
 
        """Iterate over standard values."""
 
        return iter(self._stdvalues)
 

	
 
    def get(self,
 
            key: MetaValueEnum,
 
            default_key: Optional[MetaValueEnum]=None,
 
    ) -> Optional[MetaValueEnum]:
 
        """Return self[key], or a default fallback if that doesn't exist.
 

	
 
        default_key is another key to look up, *not* a default value to return.
 
        This helps ensure you always get a standard value.
 
        """
 
        try:
 
            return self[key]
 
        except KeyError:
 
            if default_key is None:
 
                return None
 
            else:
 
                return self[default_key]
 

	
 

	
 
### HOOK SUBCLASSES
 

	
 
class _PostingHook(TransactionHook, metaclass=abc.ABCMeta):
 
    TXN_DATE_RANGE: _GenericRange = _GenericRange(DEFAULT_START_DATE, DEFAULT_STOP_DATE)
 

	
 
    def __init_subclass__(cls) -> None:
 
        cls.HOOK_GROUPS = cls.HOOK_GROUPS.union(['posting'])
 

	
 
    def _run_on_txn(self, txn: Transaction) -> bool:
 
        return txn.date in self.TXN_DATE_RANGE
 

	
 
    def _run_on_post(self, txn: Transaction, post: data.Posting) -> bool:
 
        return True
 

	
 
    def run(self, txn: Transaction) -> errormod.Iter:
 
        if self._run_on_txn(txn):
 
            for post in data.iter_postings(txn):
 
                if self._run_on_post(txn, post):
 
                    yield from self.post_run(txn, post)
 

	
 
    @abc.abstractmethod
 
    def post_run(self, txn: Transaction, post: data.Posting) -> errormod.Iter: ...
 

	
 

	
 
class _NormalizePostingMetadataHook(_PostingHook):
 
    """Base class to normalize posting metadata from an enum."""
 
    # This class provides basic functionality to filter postings, normalize
 
    # metadata values, and set default values.
 
    METADATA_KEY: MetaKey
 
    VALUES_ENUM: MetadataEnum
 

	
 
    def __init_subclass__(cls) -> None:
 
        super().__init_subclass__()
 
        cls.METADATA_KEY = cls.VALUES_ENUM.key
 
        cls.HOOK_GROUPS = cls.HOOK_GROUPS.union(['metadata', cls.METADATA_KEY])
 

	
 
    # If the posting does not specify METADATA_KEY, the hook calls
 
    # _default_value to get a default. This method should either return
 
    # a value string from METADATA_ENUM, or else raise InvalidMetadataError.
 
    # This base implementation does the latter.
 
    def _default_value(self, txn: Transaction, post: data.Posting) -> MetaValueEnum:
 
        raise errormod.InvalidMetadataError(txn, post, self.METADATA_KEY)
 
        raise errormod.InvalidMetadataError(txn, self.METADATA_KEY, None, post)
 

	
 
    def post_run(self, txn: Transaction, post: data.Posting) -> errormod.Iter:
 
        source_value = post.meta.get(self.METADATA_KEY)
 
        set_value = source_value
 
        error: Optional[errormod.Error] = None
 
        if source_value is None:
 
            try:
 
                set_value = self._default_value(txn, post)
 
            except errormod.Error as error_:
 
                error = error_
 
        else:
 
            try:
 
                set_value = self.VALUES_ENUM[source_value]
 
            except KeyError:
 
                error = errormod.InvalidMetadataError(
 
                    txn, post, self.METADATA_KEY, source_value,
 
                    txn, self.METADATA_KEY, source_value, post,
 
                )
 
        if error is None:
 
            post.meta[self.METADATA_KEY] = set_value
 
        else:
 
            yield error
conservancy_beancount/plugin/meta_entity.py
Show inline comments
 
"""meta_entity - Validate entity metadata"""
 
# Copyright © 2020  Brett Smith
 
#
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU Affero General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU Affero General Public License for more details.
 
#
 
# You should have received a copy of the GNU Affero General Public License
 
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 

	
 
import re
 

	
 
from . import core
 
from .. import data
 
from .. import errors as errormod
 
from ..beancount_types import (
 
    Transaction,
 
)
 

	
 
class MetaEntity(core.TransactionHook):
 
    METADATA_KEY = 'entity'
 
    HOOK_GROUPS = frozenset(['posting', 'metadata', METADATA_KEY])
 
    ENTITY_RE = re.compile(r'^[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*$')
 

	
 
    def run(self, txn: Transaction) -> errormod.Iter:
 
        txn_entity = txn.meta.get(self.METADATA_KEY)
 
        if txn_entity is None:
 
            txn_entity_ok = None
 
        elif isinstance(txn_entity, str):
 
            txn_entity_ok = bool(self.ENTITY_RE.match(txn_entity))
 
        else:
 
            txn_entity_ok = False
 
        if txn_entity_ok is False:
 
            yield errormod.InvalidEntityError(txn, value=txn_entity)
 
            yield errormod.InvalidMetadataError(txn, self.METADATA_KEY, txn_entity)
 
        for post in data.iter_postings(txn):
 
            if post.account.is_under('Assets', 'Liabilities'):
 
                continue
 
            entity = post.meta.get(self.METADATA_KEY)
 
            if entity is None:
 
                yield errormod.InvalidEntityError(txn, post)
 
                yield errormod.InvalidMetadataError(txn, self.METADATA_KEY, entity, post)
 
            elif entity is txn_entity:
 
                pass
 
            elif not self.ENTITY_RE.match(entity):
 
                yield errormod.InvalidEntityError(txn, post, value=entity)
 
                yield errormod.InvalidMetadataError(txn, self.METADATA_KEY, entity, post)
conservancy_beancount/plugin/meta_income_type.py
Show inline comments
 
"""meta_income_type - Validate income-type metadata"""
 
# Copyright © 2020  Brett Smith
 
#
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU Affero General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU Affero General Public License for more details.
 
#
 
# You should have received a copy of the GNU Affero General Public License
 
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 

	
 
from . import core
 
from .. import data
 
from .. import errors as errormod
 
from ..beancount_types import (
 
    MetaValueEnum,
 
    Transaction,
 
)
 

	
 
class MetaIncomeType(core._NormalizePostingMetadataHook):
 
    VALUES_ENUM = core.MetadataEnum('income-type', {
 
        'Donations',
 
        'Payable-Derecognition',
 
        'RBI',
 
        'UBTI',
 
    })
 
    DEFAULT_VALUES = {
 
        'Income:Donations': 'Donations',
 
        'Income:Honoraria': 'RBI',
 
        'Income:Interest': 'RBI',
 
        'Income:Interest:Dividend': 'RBI',
 
        'Income:Royalties': 'RBI',
 
        'Income:Sales': 'RBI',
 
        'Income:SoftwareDevelopment': 'RBI',
 
        'Income:TrademarkLicensing': 'RBI',
 
        'UnearnedIncome:Conferences:Registrations': 'RBI',
 
        'UnearnedIncome:MatchPledges': 'Donations',
 
    }
 

	
 
    def _run_on_post(self, txn: Transaction, post: data.Posting) -> bool:
 
        return post.account.is_income()
 

	
 
    def _default_value(self, txn: Transaction, post: data.Posting) -> MetaValueEnum:
 
        try:
 
            return self.DEFAULT_VALUES[post.account]
 
        except KeyError:
 
            raise errormod.InvalidMetadataError(txn, post, self.METADATA_KEY) from None
 
            raise errormod.InvalidMetadataError(txn, self.METADATA_KEY, None, post) from None
conservancy_beancount/plugin/meta_project.py
Show inline comments
 
"""meta_project - Validate project metadata"""
 
# Copyright © 2020  Brett Smith
 
#
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU Affero General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU Affero General Public License for more details.
 
#
 
# You should have received a copy of the GNU Affero General Public License
 
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 

	
 
from pathlib import Path
 

	
 
import yaml
 
import yaml.error
 

	
 
from . import core
 
from .. import config as configmod
 
from .. import data
 
from .. import errors as errormod
 
from ..beancount_types import (
 
    MetaValueEnum,
 
    Transaction,
 
)
 

	
 
from typing import (
 
    Any,
 
    Dict,
 
    Optional,
 
    Set,
 
)
 

	
 
class MetaProject(core._NormalizePostingMetadataHook):
 
    DEFAULT_PROJECT = 'Conservancy'
 
    PROJECT_DATA_PATH = Path('Projects', 'project-data.yml')
 
    VALUES_ENUM = core.MetadataEnum('project', {DEFAULT_PROJECT})
 

	
 
    def __init__(self, config: configmod.Config, source_path: Path=PROJECT_DATA_PATH) -> None:
 
        repo_path = config.repository_path()
 
        if repo_path is None:
 
            return self._config_error("no repository configured")
 
        project_data_path = repo_path / source_path
 
        source = {'filename': str(project_data_path)}
 
        try:
 
            with project_data_path.open() as yaml_file:
 
                project_data: Dict[str, Dict[str, Any]] = yaml.safe_load(yaml_file)
 
            names: Set[MetaValueEnum] = {self.DEFAULT_PROJECT}
 
            aliases: Dict[MetaValueEnum, MetaValueEnum] = {}
 
            for key, params in project_data.items():
 
                name = params.get('accountName', key)
 
                names.add(name)
 
                human_name = params.get('humanName', name)
 
                if name != human_name:
 
                    aliases[human_name] = name
 
                if name != key:
 
                    aliases[key] = name
 
        except AttributeError:
 
            self._config_error("loaded YAML data not in project-data format", project_data_path)
 
        except OSError as error:
 
            self._config_error(error.strerror, project_data_path)
 
        except yaml.error.YAMLError as error:
 
            self._config_error(error.args[0] or "YAML load error", project_data_path)
 
        else:
 
            self.VALUES_ENUM = core.MetadataEnum(self.METADATA_KEY, names, aliases)
 

	
 
    def _config_error(self, msg: str, filename: Optional[Path]=None):
 
        source = {}
 
        if filename is not None:
 
            source['filename'] = str(filename)
 
        raise errormod.ConfigurationError(
 
            "cannot load project data: " + msg,
 
            source=source,
 
        )
 

	
 
    def _run_on_post(self, txn: Transaction, post: data.Posting) -> bool:
 
        return post.account.is_under('Assets', 'Liabilities') is None
 

	
 
    def _default_value(self, txn: Transaction, post: data.Posting) -> MetaValueEnum:
 
        if post.account.is_under(
 
                'Accrued:VacationPayable',
 
                'Expenses:Payroll',
 
        ):
 
            return self.DEFAULT_PROJECT
 
        else:
 
            raise errormod.InvalidMetadataError(txn, post, self.METADATA_KEY)
 

	
 
            raise errormod.InvalidMetadataError(txn, self.METADATA_KEY, None, post)
tests/test_meta_entity.py
Show inline comments
 
"""Test validation of entity metadata"""
 
# Copyright © 2020  Brett Smith
 
#
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU Affero General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU Affero General Public License for more details.
 
#
 
# You should have received a copy of the GNU Affero General Public License
 
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 

	
 
import pytest
 

	
 
from . import testutil
 

	
 
from conservancy_beancount.plugin import meta_entity
 

	
 
VALID_VALUES = {
 
    'Smith-Alex',
 
    'Company19',
 
    'boyd-danah',
 
    'B-van-der-A',
 
}
 

	
 
INVALID_VALUES = {
 
    '-foo',
 
    'foo-',
 
    '-',
 
    'Überentity',
 
    'Alex Smith',
 
    ' ',
 
    '',
 
}
 

	
 
TEST_KEY = 'entity'
 

	
 
@pytest.fixture(scope='module')
 
def hook():
 
    config = testutil.TestConfig()
 
    return meta_entity.MetaEntity(config)
 

	
 
@pytest.mark.parametrize('src_value', VALID_VALUES)
 
def test_valid_values_on_postings(hook, src_value):
 
    txn = testutil.Transaction(postings=[
 
        ('Assets:Cash', -25),
 
        ('Expenses:General', 25, {TEST_KEY: src_value}),
 
    ])
 
    assert not any(hook.run(txn))
 

	
 
@pytest.mark.parametrize('src_value', INVALID_VALUES)
 
def test_invalid_values_on_postings(hook, src_value):
 
    txn = testutil.Transaction(postings=[
 
        ('Assets:Cash', -25),
 
        ('Expenses:General', 25, {TEST_KEY: src_value}),
 
    ])
 
    errors = list(hook.run(txn))
 
    assert len(errors) == 1
 
    assert errors[0].message == "Expenses:General entity malformed: {}".format(src_value)
 
    assert errors[0].message == "Expenses:General has invalid entity: {}".format(src_value)
 

	
 
@pytest.mark.parametrize('src_value', VALID_VALUES)
 
def test_valid_values_on_transactions(hook, src_value):
 
    txn = testutil.Transaction(**{TEST_KEY: src_value}, postings=[
 
        ('Assets:Cash', -25),
 
        ('Expenses:General', 25),
 
    ])
 
    assert not any(hook.run(txn))
 

	
 
@pytest.mark.parametrize('src_value', INVALID_VALUES)
 
def test_invalid_values_on_transactions(hook, src_value):
 
    txn = testutil.Transaction(**{TEST_KEY: src_value}, postings=[
 
        ('Assets:Cash', -25),
 
        ('Expenses:General', 25),
 
    ])
 
    errors = list(hook.run(txn))
 
    assert 1 <= len(errors) <= 2
 
    assert all(error.message == "transaction entity malformed: {}".format(src_value)
 
    assert all(error.message == "transaction has invalid entity: {}".format(src_value)
 
               for error in hook.run(txn))
 

	
 
@pytest.mark.parametrize('account,required', [
 
    ('Accrued:AccountsReceivable', True),
 
    ('Assets:Cash', False),
 
    ('Expenses:General', True),
 
    ('Income:Donations', True),
 
    ('Liabilities:CreditCard', False),
 
    ('UnearnedIncome:Donations', True),
 
])
 
def test_which_accounts_required_on(hook, account, required):
 
    txn = testutil.Transaction(postings=[
 
        ('Assets:Checking', 25),
 
        (account, 25),
 
    ])
 
    errors = list(hook.run(txn))
 
    if not required:
 
        assert not errors
 
    else:
 
        assert errors
 
        assert any(error.message == "{} missing entity".format(account)
 
                   for error in errors)
0 comments (0 inline, 0 general)