From ea0e3d3a73b7b167ba0f4690625fc2580de1419d 2018-04-03 15:21:12 From: Brett Smith Date: 2018-04-03 15:21:12 Subject: [PATCH] dynload: Pre-configure objects and skip unconfigured ones. Now that all importers and hooks receive the configuration object, they can tell us at initialization time whether or not they actually have enough configuration to be useful. Initialize them immediately on loading, and only use importers and hooks that are actually configured. --- diff --git a/TODO.rst b/TODO.rst index 306378707349ab0672f62976c6b4bb98033f9139..72d9e577c263e794c360b7b996b123080285e649 100644 --- a/TODO.rst +++ b/TODO.rst @@ -18,7 +18,6 @@ New importers ------------- * Stripe import via API - * This is going to require giving importer objects the configuration so they can read things like API keys. * YourCause * Network for Good diff --git a/import2ledger/__main__.py b/import2ledger/__main__.py index 53672ac9fb45bc8aa1c5378c1fde72a3f0f260fa..27af14b6059880ae8887214c2f15a5cbce525b4a 100644 --- a/import2ledger/__main__.py +++ b/import2ledger/__main__.py @@ -11,8 +11,8 @@ logger = logging.getLogger('import2ledger') class FileImporter: def __init__(self, config, stdout): self.config = config - self.importers = [importer(config) for importer in importers.load_all()] - self.hooks = [hook(config) for hook in hooks.load_all()] + self.importers = importers.load_all(config) + self.hooks = hooks.load_all(config) self.stdout = stdout def import_file(self, in_file, in_path=None): diff --git a/import2ledger/dynload.py b/import2ledger/dynload.py index ca0622d4d76b4cc3a30154e8faa96a174e91b397..01f616e2b045e7818962d07b7658c25a6b0000e1 100644 --- a/import2ledger/dynload.py +++ b/import2ledger/dynload.py @@ -2,6 +2,8 @@ import importlib import logging import pathlib +from . import errors + logger = logging.getLogger('import2ledger') def load_modules(src_dir_path): @@ -23,8 +25,11 @@ def module_contents(module): for name in dir(module): yield name, getattr(module, name) -def submodule_items_named(file_path, name_test): +def submodule_items_named(file_path, name_test, config): for module in load_modules(pathlib.Path(file_path).parent): for name, item in module_contents(module): if name_test(name): - yield item + try: + yield item(config) + except errors.NotConfiguredError as error: + logger.info("failed to load %s: %s", name, error.strerror) diff --git a/import2ledger/errors.py b/import2ledger/errors.py index 4d2b81ee0291d32b0c4428b722da60e814635bbc..8a10afcac8e50f5bca9598aa186f7ab8e482b370 100644 --- a/import2ledger/errors.py +++ b/import2ledger/errors.py @@ -12,6 +12,10 @@ class UserInputConfigurationError(UserInputError): pass +class NotConfiguredError(UserInputConfigurationError): + pass + + class UserInputFileError(UserInputError): def __init__(self, strerror, path): super().__init__(strerror, path) diff --git a/import2ledger/hooks/__init__.py b/import2ledger/hooks/__init__.py index 87a345f4c98f486bfb856d64d475198b909a30ad..2dd4a4b32ab893301d036b767bc7028f67817c03 100644 --- a/import2ledger/hooks/__init__.py +++ b/import2ledger/hooks/__init__.py @@ -23,7 +23,8 @@ HOOK_KINDS = enum.Enum('HOOK_KINDS', [ 'OUTPUT', ]) -def load_all(): - hooks = list(dynload.submodule_items_named(__file__, operator.methodcaller('endswith', 'Hook'))) +def load_all(config): + hooks = list(dynload.submodule_items_named( + __file__, operator.methodcaller('endswith', 'Hook'), config)) hooks.sort(key=operator.attrgetter('KIND.value')) return hooks diff --git a/import2ledger/importers/__init__.py b/import2ledger/importers/__init__.py index 68e49acca458a5e613704cfd20d1f39aaa04e1cd..faf5aa78be7ea793fe23c81090babbcc947b87d5 100644 --- a/import2ledger/importers/__init__.py +++ b/import2ledger/importers/__init__.py @@ -2,5 +2,6 @@ import operator from .. import dynload -def load_all(): - return dynload.submodule_items_named(__file__, operator.methodcaller('endswith', 'Importer')) +def load_all(config): + return list(dynload.submodule_items_named( + __file__, operator.methodcaller('endswith', 'Importer'), config)) diff --git a/tests/date_hooks.py b/tests/date_hooks.py new file mode 100644 index 0000000000000000000000000000000000000000..85c018892a28accf27581afa9a96600bd6239404 --- /dev/null +++ b/tests/date_hooks.py @@ -0,0 +1,23 @@ +import datetime + +class DateHookTestBase: + DATE_FMT_ISO = '%Y-%m-%d' + DATE_FMT_LEDGER = '%d/%m/%Y' + + def _format_config_date(self, section, key, value, date_fmt): + if value is not None: + section[key] = value.strftime(date_fmt) + + def new_config( + self, + fmt=DATE_FMT_ISO, + default_date=datetime.date(2017, 3, 1), + start_date=datetime.date(2017, 2, 1), + end_date=datetime.date(2017, 4, 1) + ): + section = {'date format': fmt.replace('%', '%%')} + self._format_config_date(section, 'default date', default_date, fmt) + self._format_config_date(section, 'import start date', start_date, fmt) + self._format_config_date(section, 'import end date', end_date, fmt) + return {'Dates': section} + diff --git a/tests/test_hooks.py b/tests/test_hooks.py index 5c3bee8a884e581b306fe4aaed2ceab4dbb851f9..fe787922501eadab62a7e11539a1a88e24eaa844 100644 --- a/tests/test_hooks.py +++ b/tests/test_hooks.py @@ -7,17 +7,28 @@ import pytest from import2ledger import hooks from import2ledger.hooks import add_entity, default_date, filter_by_date, ledger_entry +from . import Config +from . import date_hooks + +def _run_order_test(config, expected_order): + all_hooks = list(hooks.load_all(config)) + positions = {type(hook): index for index, hook in enumerate(all_hooks)} + actual_order = list(sorted(expected_order, key=positions.__getitem__)) + assert actual_order == expected_order + +def test_load_no_config_needed(): + return _run_order_test(Config(), [ + add_entity.AddEntityHook, + ]) + def test_load_all(): - all_hooks = list(hooks.load_all()) - positions = {hook: index for index, hook in enumerate(all_hooks)} - expected_order = [ + config_dict = date_hooks.DateHookTestBase().new_config() + return _run_order_test(Config(config_dict), [ default_date.DefaultDateHook, add_entity.AddEntityHook, filter_by_date.FilterByDateHook, ledger_entry.LedgerEntryHook, - ] - actual_order = list(sorted(expected_order, key=positions.__getitem__)) - assert actual_order == expected_order + ]) class DateRangeConfig: def __init__(self, start_date=None, end_date=None): diff --git a/tests/test_importers.py b/tests/test_importers.py index c286c6b90a6754758813739362ff4ee5b1207c2a..39c71df260a3c311973a930662771a05a1ff2f39 100644 --- a/tests/test_importers.py +++ b/tests/test_importers.py @@ -12,6 +12,8 @@ from import2ledger import errors, importers, strparse from . import Config, DATA_DIR class TestImporters: + FULL_CONFIG = Config({}) + with pathlib.Path(DATA_DIR, 'imports.yml').open() as yaml_file: test_data = yaml.load(yaml_file) for test in test_data: @@ -46,6 +48,6 @@ class TestImporters: assert actual == expected def test_loader(self): - all_importers = list(importers.load_all()) + all_importers = {type(importer) for importer in importers.load_all(self.FULL_CONFIG)} for test in self.test_data: assert test['importer'] in all_importers