Changeset - 499f18ff623c
Parent rev.
Child rev.
[Not reviewed]
0 3 0
Brett Smith - 4 years ago 2020-04-01 17:38:37
brettcsmith@brettcsmith.org
meta_entity: Adjust what entities are allowed based on today's books.

See the comments throughout for more discussion about what cases are
or aren't allowed, and why.
3 files changed with 52 insertions and 3 deletions:
conservancy_beancount/plugin/meta_entity.py
22
2
setup.py
1
tests/test_meta_entity.py
29
1
0 comments (0 inline, 0 general)
conservancy_beancount/plugin/meta_entity.py
➞
Show inline comments
 
"""meta_entity - Validate entity metadata"""
 
# Copyright © 2020  Brett Smith
 
#
 
# This program is free software: you can redistribute it and/or modify
 
# it under the terms of the GNU Affero General Public License as published by
 
# the Free Software Foundation, either version 3 of the License, or
 
# (at your option) any later version.
 
#
 
# This program is distributed in the hope that it will be useful,
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 
# GNU Affero General Public License for more details.
 
#
 
# You should have received a copy of the GNU Affero General Public License
 
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
 

			




	
	
	
		
 
import re


			




	
	
	
		
 
# Type stubs aren't available for regex.


			




	
	
	
		
 
# Fortunately, we're using it in a way that's API-compatible with the re


			




	
	
	
		
 
# module. We mitigate the lack of type stubs by providing type declarations


			




	
	
	
		
 
# for returned objects. This way, the only thing that isn't type checked are


			




	
	
	
		
 
# the calls to regex functions.


			




	
	
	
		
 
import regex  # type:ignore[import]


			




	
	
	
		
 

			




	
	
	
		
 
from . import core


			




	
	
	
		
 
from .. import data


			




	
	
	
		
 
from .. import errors as errormod


			




	
	
	
		
 
from ..beancount_types import (


			




	
	
	
		
 
    Transaction,


			




	
	
	
		
 
)


			




	
	
	
		
 

			




	
	
	
		
 
from typing import (


			




	
	
	
		
 
    Pattern,


			




	
	
	
		
 
)


			




	
	
	
		
 

			




	
	
	
		
 
class MetaEntity(core.TransactionHook):


			




	
	
	
		
 
    METADATA_KEY = 'entity'


			




	
	
	
		
 
    HOOK_GROUPS = frozenset(['posting', 'metadata', METADATA_KEY])


			




	
	
	
		
 
    ENTITY_RE = re.compile(r'^[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*$')


			




	
	
	
		
 

			




	
	
	
		
 
    # alnum is the set of characters we always accept in entity metadata:


			




	
	
	
		
 
    # letters and digits, minus the Latin 1 supplement (i.e., Roman letters


			




	
	
	
		
 
    # with diacritics: áÁàÀâÂåÅäÄãà çÇ ðÐ ñÑ øØ ß etc.)


			




	
	
	
		
 
    # See the tests for specific cases.


			




	
	
	
		
 
    alnum = r'\p{Letter}\p{Digit}--\p{Block=Latin_1_Supplement}'


			




	
	
	
		
 
    # A regexp that would be reasonably stricter would be:


			




	
	
	
		
 
    #   f'^[{alnum}][.{alnum}]*(?:-[.{alnum}])*$'


			




	
	
	
		
 
    # However, current producers fail that regexp in a few different ways.


			




	
	
	
		
 
    # See the tests for specific cases.


			




	
	
	
		
 
    ENTITY_RE: Pattern[str] = regex.compile(f'^[{alnum}][-.{alnum}]*$', regex.VERSION1)


			




	
	
	
		
 
    del alnum


			




	
	
	
		
 

			




	
	
	
		
 
    def run(self, txn: Transaction) -> errormod.Iter:


			




	
	
	
		
 
        txn_entity = txn.meta.get(self.METADATA_KEY)


			




	
	
	
		
 
        if txn_entity is None:


			




	
	
	
		
 
            txn_entity_ok = None


			




	
	
	
		
 
        elif isinstance(txn_entity, str):


			




	
	
	
		
 
            txn_entity_ok = bool(self.ENTITY_RE.match(txn_entity))


			




	
	
	
		
 
        else:


			




	
	
	
		
 
            txn_entity_ok = False


			




	
	
	
		
 
        if txn_entity_ok is False:


			




	
	
	
		
 
            yield errormod.InvalidMetadataError(txn, self.METADATA_KEY, txn_entity)


			




	
	
	
		
 
        for post in data.iter_postings(txn):


			




	
	
	
		
 
            if post.account.is_under('Assets', 'Equity', 'Liabilities'):


			




	
	
	
		
 
                continue


			




	
	
	
		
 
            entity = post.meta.get(self.METADATA_KEY)


			




	
	
	
		
 
            if entity is None:


			




	
	
	
		
 
                yield errormod.InvalidMetadataError(txn, self.METADATA_KEY, entity, post)


			




	
	
	
		
 
            elif entity is txn_entity:


			




	
	
	
		
 
                pass


			




	
	
	
		
 
            elif not self.ENTITY_RE.match(entity):


			




	
	
	
		
 
                yield errormod.InvalidMetadataError(txn, self.METADATA_KEY, entity, post)


			




        

    


    
    

    

        

                

                    setup.py
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
	
		
 
#!/usr/bin/env python3


			




	
	
	
		
 

			




	
	
	
		
 
from setuptools import setup


			




	
	
	
		
 

			




	
	
	
		
 
setup(


			




	
	
	
		
 
    name='conservancy_beancount',


			




	
	
	
		
 
    description="Plugin, library, and reports for reading Conservancy's books",


			




	
	
	
		
 
    version='0.1',


			




	
	
	
		
 
    author='Software Freedom Conservancy',


			




	
	
	
		
 
    author_email='info@sfconservancy.org',


			




	
	
	
		
 
    license='GNU AGPLv3+',


			




	
	
	
		
 

			




	
	
	
		
 
    install_requires=[


			




	
	
	
		
 
        'beancount>=2.2',


			




	
	
	
		
 
        'regex',


			




	
	
	
		
 
        'rt>=2.0',


			




	
	
	
		
 
    ],


			




	
	
	
		
 
    setup_requires=[


			




	
	
	
		
 
        'pytest-mypy',


			




	
	
	
		
 
        'pytest-runner',


			




	
	
	
		
 
    ],


			




	
	
	
		
 
    tests_require=[


			




	
	
	
		
 
        'mypy>=0.770',


			




	
	
	
		
 
        'pytest',


			




	
	
	
		
 
    ],


			




	
	
	
		
 

			




	
	
	
		
 
    packages=['conservancy_beancount'],


			




	
	
	
		
 
    entry_points={},


			




	
	
	
		
 
)


			




        

    


    
    

    

        

                

                    tests/test_meta_entity.py
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	
	
	
		
 
"""Test validation of entity metadata"""


			




	
	
	
		
 
# Copyright © 2020  Brett Smith


			




	
	
	
		
 
#


			




	
	
	
		
 
# This program is free software: you can redistribute it and/or modify


			




	
	
	
		
 
# it under the terms of the GNU Affero General Public License as published by


			




	
	
	
		
 
# the Free Software Foundation, either version 3 of the License, or


			




	
	
	
		
 
# (at your option) any later version.


			




	
	
	
		
 
#


			




	
	
	
		
 
# This program is distributed in the hope that it will be useful,


			




	
	
	
		
 
# but WITHOUT ANY WARRANTY; without even the implied warranty of


			




	
	
	
		
 
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the


			




	
	
	
		
 
# GNU Affero General Public License for more details.


			




	
	
	
		
 
#


			




	
	
	
		
 
# You should have received a copy of the GNU Affero General Public License


			




	
	
	
		
 
# along with this program.  If not, see <https://www.gnu.org/licenses/>.


			




	
	
	
		
 

			




	
	
	
		
 
import pytest


			




	
	
	
		
 

			




	
	
	
		
 
from . import testutil


			




	
	
	
		
 

			




	
	
	
		
 
from conservancy_beancount.plugin import meta_entity


			




	
	
	
		
 

			




	
	
	
		
 
VALID_VALUES = {


			




	
	
	
		
 
    # Classic entity: LastName-FirstName


			




	
	
	
		
 
    'Smith-Alex',


			




	
	
	
		
 
    # Various people and companies have one-word names


			




	
	
	
		
 
    # Digits are allowed, as part of a name or standalone


			




	
	
	
		
 
    'Company19',


			




	
	
	
		
 
    'Company-19',


			




	
	
	
		
 
    # No case requirements


			




	
	
	
		
 
    'boyd-danah',


			




	
	
	
		
 
    # No limit on the number of parts of the name


			




	
	
	
		
 
    'B-van-der-A',


			




	
	
	
		
 
    # Names that have no ASCII are allowed, with or without dash separators


			




	
	
	
		
 
    '田中流星',


			




	
	
	
		
 
    '田中-流星',


			




	
	
	
		
 
    'スミスダコタ',


			




	
	
	
		
 
    'スミス-ダコタ',


			




	
	
	
		
 
    'Яшин-Данила',


			




	
	
	
		
 
    # The PayPal importer produces . in entity metadata


			




	
	
	
		
 
    'Du-Bois-W.-E.-B.',


			




	
	
	
		
 
    # import2ledger produces entities that end with -


			




	
	
	
		
 
    # That's probably a bug, but allow it for now.


			




	
	
	
		
 
    'foo-',


			




	
	
	
		
 
}


			




	
	
	
		
 

			




	
	
	
		
 
INVALID_VALUES = {


			




	
	
	
		
 
    # Starting with a - is not allowed


			




	
	
	
		
 
    '-foo',


			




	
	
	
		
 
    'foo-',


			




	
	
	
		
 
    '-',


			




	
	
	
		
 
    # Names that can be reduced to ASCII should be


			




	
	
	
		
 
    # Producers should change this to Uberentity or Ueberentity


			




	
	
	
		
 
    # I am not wild about this rule and would like to relax it—it's mostly


			




	
	
	
		
 
    # based on an expectation that entities are typed in by a human. That's true


			




	
	
	
		
 
    # less and less and it seems like we should reduce the amount of mangling


			




	
	
	
		
 
    # producers are expected to do. But it's the rule for today.


			




	
	
	
		
 
    'Überentity',


			




	
	
	
		
 
    # Whitespace is never allowed


			




	
	
	
		
 
    'Alex Smith',


			




	
	
	
		
 
    '田中 流星',


			




	
	
	
		
 
    'スミス ダコタ',


			




	
	
	
		
 
    'Яшин Данила',


			




	
	
	
		
 
    ' ',


			




	
	
	
		
 
    # An empty string is not valid


			




	
	
	
		
 
    '',


			




	
	
	
		
 
}


			




	
	
	
		
 

			




	
	
	
		
 
TEST_KEY = 'entity'


			




	
	
	
		
 

			




	
	
	
		
 
@pytest.fixture(scope='module')


			




	
	
	
		
 
def hook():


			




	
	
	
		
 
    config = testutil.TestConfig()


			




	
	
	
		
 
    return meta_entity.MetaEntity(config)


			




	
	
	
		
 

			




	
	
	
		
 
@pytest.mark.parametrize('src_value', VALID_VALUES)


			




	
	
	
		
 
def test_valid_values_on_postings(hook, src_value):


			




	
	
	
		
 
    txn = testutil.Transaction(postings=[


			




	
	
	
		
 
        ('Assets:Cash', -25),


			




	
	
	
		
 
        ('Expenses:General', 25, {TEST_KEY: src_value}),


			




	
	
	
		
 
    ])


			




	
	
	
		
 
    assert not any(hook.run(txn))


			




	
	
	
		
 

			




	
	
	
		
 
@pytest.mark.parametrize('src_value', INVALID_VALUES)


			




	
	
	
		
 
def test_invalid_values_on_postings(hook, src_value):


			




	
	
	
		
 
    txn = testutil.Transaction(postings=[


			




	
	
	
		
 
        ('Assets:Cash', -25),


			




	
	
	
		
 
        ('Expenses:General', 25, {TEST_KEY: src_value}),


			




	
	
	
		
 
    ])


			




	
	
	
		
 
    errors = list(hook.run(txn))


			




	
	
	
		
 
    assert len(errors) == 1


			




	
	
	
		
 
    assert errors[0].message == "Expenses:General has invalid entity: {}".format(src_value)


			




	
	
	
		
 

			




	
	
	
		
 
@pytest.mark.parametrize('src_value', VALID_VALUES)


			




	
	
	
		
 
def test_valid_values_on_transactions(hook, src_value):


			




	
	
	
		
 
    txn = testutil.Transaction(**{TEST_KEY: src_value}, postings=[


			




	
	
	
		
 
        ('Assets:Cash', -25),


			




	
	
	
		
 
        ('Expenses:General', 25),


			




	
	
	
		
 
    ])


			




	
	
	
		
 
    assert not any(hook.run(txn))


			




	
	
	
		
 

			




	
	
	
		
 
@pytest.mark.parametrize('src_value', INVALID_VALUES)


			




	
	
	
		
 
def test_invalid_values_on_transactions(hook, src_value):


			




	
	
	
		
 
    txn = testutil.Transaction(**{TEST_KEY: src_value}, postings=[


			




	
	
	
		
 
        ('Assets:Cash', -25),


			




	
	
	
		
 
        ('Expenses:General', 25),


			




	
	
	
		
 
    ])


			




	
	
	
		
 
    errors = list(hook.run(txn))


			




	
	
	
		
 
    assert 1 <= len(errors) <= 2


			




	
	
	
		
 
    assert all(error.message == "transaction has invalid entity: {}".format(src_value)


			




	
	
	
		
 
               for error in hook.run(txn))


			




	
	
	
		
 

			




	
	
	
		
 
@pytest.mark.parametrize('account,required', [


			




	
	
	
		
 
    ('Accrued:AccountsReceivable', True),


			




	
	
	
		
 
    ('Assets:Cash', False),


			




	
	
	
		
 
    ('Equity:OpeningBalances', False),


			




	
	
	
		
 
    ('Expenses:General', True),


			




	
	
	
		
 
    ('Income:Donations', True),


			




	
	
	
		
 
    ('Liabilities:CreditCard', False),


			




	
	
	
		
 
    ('UnearnedIncome:Donations', True),


			




	
	
	
		
 
])


			




	
	
	
		
 
def test_which_accounts_required_on(hook, account, required):


			




	
	
	
		
 
    txn = testutil.Transaction(postings=[


			




	
	
	
		
 
        ('Assets:Checking', 25),


			




	
	
	
		
 
        (account, 25),


			




	
	
	
		
 
    ])


			




	
	
	
		
 
    errors = list(hook.run(txn))


			




	
	
	
		
 
    if not required:


			




	
	
	
		
 
        assert not errors


			




	
	
	
		
 
    else:


			




	
	
	
		
 
        assert errors


			




	
	
	
		
 
        assert any(error.message == "{} missing entity".format(account)


			




	
	
	
		
 
                   for error in errors)


			




        

    



    


    



    



      

      





    
    0 comments (0 inline, 0 general)
    





    


  

  







    


    

  


    








    
        Server instance: kapok.sfconservancy.org-4015
    
    
        This site is powered by
            Kallithea 0.6.3,
        which is
        © 2010–2020 by various authors & licensed under GPLv3.
            – Support
        – Privacy Policy