Changeset - 56b644f1db9b
Parent rev.
Child rev.
[Not reviewed]
0 2 0
Brett Smith - 4 years ago 2020-05-06 14:26:25
brettcsmith@brettcsmith.org
meta_entity: More battle testing.

See the test cases for examples of real entities in the books
that we should accept for now.
2 files changed with 81 insertions and 28 deletions:
conservancy_beancount/plugin/meta_entity.py
32
22
tests/test_meta_entity.py
49
6
0 comments (0 inline, 0 general)
conservancy_beancount/plugin/meta_entity.py
➞
Show inline comments
...
 
@@ -27,2 +27,4 @@ from .. import errors as errormod
 
from ..beancount_types import (
 
    MetaKey,
 
    MetaValue,
 
    Transaction,
...
 
@@ -31,3 +33,6 @@ from ..beancount_types import (
 
from typing import (
 
    MutableMapping,
 
    Optional,
 
    Pattern,
 
    Tuple,
 
)
...
 
@@ -38,13 +43,26 @@ class MetaEntity(core.TransactionHook):
 

			




	
	
	
		
 
    # alnum is the set of characters we always accept in entity metadata:


			




	
	
	
		
 
    # letters and digits, minus the Latin 1 supplement (i.e., Roman letters


			




	
	
	
		
 
    # with diacritics: áÁàÀâÂåÅäÄãà çÇ ðÐ ñÑ øØ ß etc.)


			




	
	
	
		
 
    # chars is the set of characters we always accept in entity metadata:


			




	
	
	
		
 
    # letters, digits, and ASCII punctuation, except `-` and the Latin 1 supplement


			




	
	
	
		
 
    # (i.e., Roman letters with diacritics: áÁàÀâÂåÅäÄãà çÇ ðÐ ñÑ øØ ß etc.)


			




	
	
	
		
 
    # See the tests for specific cases.


			




	
	
	
		
 
    alnum = r'\p{Letter}\p{Digit}--\p{Block=Latin_1_Supplement}'


			




	
	
	
		
 
    # A regexp that would be reasonably stricter would be:


			




	
	
	
		
 
    #   f'^[{alnum}][.{alnum}]*(?:-[.{alnum}])*$'


			




	
	
	
		
 
    # However, current producers fail that regexp in a few different ways.


			




	
	
	
		
 
    # See the tests for specific cases.


			




	
	
	
		
 
    ENTITY_RE: Pattern[str] = regex.compile(f'^[{alnum}][-.{alnum}]*$', regex.VERSION1)


			




	
	
	
		
 
    del alnum


			




	
	
	
		
 
    chars = r'\u0021-\u002c\u002e-\u007e\p{Letter}\p{Digit}--\p{Block=Latin_1_Supplement}'


			




	
	
	
		
 
    ENTITY_RE: Pattern[str] = regex.compile(f'^[{chars}][-{chars}]*$', regex.VERSION1)


			




	
	
	
		
 
    ANONYMOUS_RE: Pattern[str] = regex.compile(r'^[-_.?!\s]*$', regex.VERSION1)


			




	
	
	
		
 
    del chars


			




	
	
	
		
 

			




	
	
	
		
 
    def _check_entity(self,


			




	
	
	
		
 
                      meta: MutableMapping[MetaKey, MetaValue],


			




	
	
	
		
 
                      default: Optional[str]=None,


			




	
	
	
		
 
    ) -> Tuple[Optional[str], Optional[bool]]:


			




	
	
	
		
 
        entity = meta.get(self.METADATA_KEY, default)


			




	
	
	
		
 
        if entity is None:


			




	
	
	
		
 
            return None, None


			




	
	
	
		
 
        elif not isinstance(entity, str):


			




	
	
	
		
 
            return None, False


			




	
	
	
		
 
        elif self.ANONYMOUS_RE.match(entity):


			




	
	
	
		
 
            entity = 'Anonymous'


			




	
	
	
		
 
            meta[self.METADATA_KEY] = entity


			




	
	
	
		
 
            return entity, True


			




	
	
	
		
 
        else:


			




	
	
	
		
 
            return entity, self.ENTITY_RE.match(entity) is not None


			




	
	
	
		
 

			




	...
	
		
 
@@ -53,9 +71,3 @@ class MetaEntity(core.TransactionHook):


			




	
	
	
		
 
            return


			




	
	
	
		
 
        txn_entity = txn.meta.get(self.METADATA_KEY, txn.payee)


			




	
	
	
		
 
        if txn_entity is None:


			




	
	
	
		
 
            txn_entity_ok = None


			




	
	
	
		
 
        elif isinstance(txn_entity, str):


			




	
	
	
		
 
            txn_entity_ok = bool(self.ENTITY_RE.match(txn_entity))


			




	
	
	
		
 
        else:


			




	
	
	
		
 
            txn_entity_ok = False


			




	
	
	
		
 
        txn_entity, txn_entity_ok = self._check_entity(txn.meta, txn.payee)


			




	
	
	
		
 
        if txn_entity_ok is False:


			




	...
	
		
 
@@ -70,8 +82,6 @@ class MetaEntity(core.TransactionHook):


			




	
	
	
		
 
                continue


			




	
	
	
		
 
            entity = post.meta.get(self.METADATA_KEY)


			




	
	
	
		
 
            if entity is None:


			




	
	
	
		
 
                yield errormod.InvalidMetadataError(txn, self.METADATA_KEY, entity, post)


			




	
	
	
		
 
            elif entity is txn_entity:


			




	
	
	
		
 
            entity, entity_ok = self._check_entity(post.meta, txn_entity)


			




	
	
	
		
 
            if entity is txn_entity and entity is not None:


			




	
	
	
		
 
                pass


			




	
	
	
		
 
            elif not self.ENTITY_RE.match(entity):


			




	
	
	
		
 
            elif not entity_ok:


			




	
	
	
		
 
                yield errormod.InvalidMetadataError(txn, self.METADATA_KEY, entity, post)


			




        

    


    
    

    

        

                

                    tests/test_meta_entity.py
                

                

                  
                      
                        

                      ➞

                      
                        
                  

                  
                      
                  
                      
                  
                      
                  
                      
                  
                  
                

                

                    Show inline comments
                    
                

        

        

            



	...
	
		
 
@@ -39,4 +39,10 @@ VALID_VALUES = {


			




	
	
	
		
 
    'Яшин-Данила',


			




	
	
	
		
 
    # The PayPal importer produces . in entity metadata


			




	
	
	
		
 
    # Governments, using : as a hierarchy separator


			




	
	
	
		
 
    'BE',


			




	
	
	
		
 
    'US:KY',


			




	
	
	
		
 
    'CA:ON',


			




	
	
	
		
 
    # The PayPal importer allows ASCII punctuation in entity metadata


			




	
	
	
		
 
    'Du-Bois-W.-E.-B.',


			




	
	
	
		
 
    "O'Malley-Thomas",


			




	
	
	
		
 
    'O`Malley-Thomas',


			




	
	
	
		
 
    # import2ledger produces entities that end with -


			




	...
	
		
 
@@ -49,3 +55,2 @@ INVALID_VALUES = {


			




	
	
	
		
 
    '-foo',


			




	
	
	
		
 
    '-',


			




	
	
	
		
 
    # Names that can be reduced to ASCII should be


			




	...
	
		
 
@@ -58,10 +63,21 @@ INVALID_VALUES = {


			




	
	
	
		
 
    # Whitespace is never allowed


			




	
	
	
		
 
    ' ',


			




	
	
	
		
 
    'Alex Smith',


			




	
	
	
		
 
    '田中\u00A0流星',  # Non-breaking space


			




	
	
	
		
 
    # The only punctuation allowed is - and .


			




	
	
	
		
 
    'スミス_ダコタ',


			




	
	
	
		
 
    # Non-ASCII punctuation is not allowed


			




	
	
	
		
 
    'Яшин—Данила',  # em dash


			




	
	
	
		
 
    # An empty string is not valid


			




	
	
	
		
 
    'O’Malley-Thomas',  # Right-angled apostrophe


			




	
	
	
		
 
    'Du-Bois-W。-E。-B。',  # Japanese period


			




	
	
	
		
 
}


			




	
	
	
		
 

			




	
	
	
		
 
ANONYMOUS_VALUES = {


			




	
	
	
		
 
    # Values produced by various importers that should be translated to


			




	
	
	
		
 
    # Anonymous.


			




	
	
	
		
 
    '',


			




	
	
	
		
 
    ' ',


			




	
	
	
		
 
    '-',


			




	
	
	
		
 
    '--',


			




	
	
	
		
 
    '-----',


			




	
	
	
		
 
    '_',


			




	
	
	
		
 
    ' _ ',


			




	
	
	
		
 
    '.',


			




	
	
	
		
 
}


			




	...
	
		
 
@@ -83,2 +99,11 @@ def test_valid_values_on_postings(hook, src_value):


			




	
	
	
		
 

			




	
	
	
		
 
@pytest.mark.parametrize('src_value', ANONYMOUS_VALUES)


			




	
	
	
		
 
def test_anonymous_values_on_postings(hook, src_value):


			




	
	
	
		
 
    txn = testutil.Transaction(postings=[


			




	
	
	
		
 
        ('Assets:Cash', -25),


			




	
	
	
		
 
        ('Expenses:General', 25, {TEST_KEY: src_value}),


			




	
	
	
		
 
    ])


			




	
	
	
		
 
    assert not any(hook.run(txn))


			




	
	
	
		
 
    assert txn.postings[-1].meta[TEST_KEY] == 'Anonymous'


			




	
	
	
		
 

			




	
	
	
		
 
@pytest.mark.parametrize('src_value', INVALID_VALUES)


			




	...
	
		
 
@@ -101,2 +126,11 @@ def test_valid_values_on_transactions(hook, src_value):


			




	
	
	
		
 

			




	
	
	
		
 
@pytest.mark.parametrize('src_value', ANONYMOUS_VALUES)


			




	
	
	
		
 
def test_anonymous_values_on_transactions(hook, src_value):


			




	
	
	
		
 
    txn = testutil.Transaction(**{TEST_KEY: src_value}, postings=[


			




	
	
	
		
 
        ('Assets:Cash', -25),


			




	
	
	
		
 
        ('Expenses:General', 25),


			




	
	
	
		
 
    ])


			




	
	
	
		
 
    assert not any(hook.run(txn))


			




	
	
	
		
 
    assert txn.meta[TEST_KEY] == 'Anonymous'


			




	
	
	
		
 

			




	
	
	
		
 
@pytest.mark.parametrize('src_value', INVALID_VALUES)


			




	...
	
		
 
@@ -120,2 +154,11 @@ def test_valid_values_on_payee(hook, src_value):


			




	
	
	
		
 

			




	
	
	
		
 
@pytest.mark.parametrize('src_value', ANONYMOUS_VALUES)


			




	
	
	
		
 
def test_anonymous_values_on_payee(hook, src_value):


			




	
	
	
		
 
    txn = testutil.Transaction(payee=src_value, postings=[


			




	
	
	
		
 
        ('Assets:Cash', -25),


			




	
	
	
		
 
        ('Expenses:General', 25),


			




	
	
	
		
 
    ])


			




	
	
	
		
 
    assert not any(hook.run(txn))


			




	
	
	
		
 
    assert txn.meta[TEST_KEY] == 'Anonymous'


			




	
	
	
		
 

			




	
	
	
		
 
@pytest.mark.parametrize('src_value', INVALID_VALUES)


			




        

    



    


    



    



      

      





    
    0 comments (0 inline, 0 general)
    





    


  

  







    


    

  


    








    
        Server instance: kapok.sfconservancy.org-27437
    
    
        This site is powered by
            Kallithea 0.6.3,
        which is
        © 2010–2020 by various authors & licensed under GPLv3.
            – Support
        – Privacy Policy