NPO-Accounting/conservancy_beancount Changeset - 69f3e4ee6eeb

Changeset - 69f3e4ee6eeb

Parent rev.

Child rev.

[Not reviewed]

0 2 0

Brett Smith - 3 years ago 2021-03-15 13:56:13
brettcsmith@brettcsmith.org

query: Add a hint for the TypeError `unhashable type: 'set'`.

2 files changed with 27 insertions and 10 deletions:

conservancy_beancount/reports/query.py

setup.py

0 comments (0 inline, 0 general)

conservancy_beancount/reports/query.py

➞

Show inline comments

@@ ... / @@ -367,401 +367,418 @@ class RTTicket(bc_query_compile.EvalFunction): @@
             if field_value is None:
                 pass
             elif isinstance(field_value, list):
                 retval.update(field_value)
             else:
                 retval.add(field_value)
         return retval
 class StrMeta(bc_query_env.AnyMeta):
     """Looks up metadata like AnyMeta, then always returns a string."""
     def __init__(self, operands: List[bc_query_compile.EvalNode]) -> None:
         super(bc_query_env.AnyMeta, self).__init__(operands, str)
     def __call__(self, context: PostingContext) -> str:
         raw_value = super().__call__(context)
         if raw_value is None:
             return ''
         else:
             return str(raw_value)
 class AggregateSet(bc_query_compile.EvalAggregator):
     """Filter argument values that aren't unique."""
     __intypes__ = [object]
     def __init__(self, operands: List[bc_query_compile.EvalNode]) -> None:
        super().__init__(operands, set)
     def allocate(self, allocator: bc_query_execute.Allocator) -> None:
         """Allocate and save an index handle into result storage."""
         self.handle = allocator.allocate()
     def initialize(self, store: Store) -> None:
         """Prepare result storage for a new aggregation."""
         store[self.handle] = self.dtype()
         # self.dtype() is our return type, aka the second argument to __init__
         # above, aka the annotated return type of __call__.
     def update(self, store: Store, context: PostingContext) -> None:
         """Update existing storage with new result data."""
         value, = self.eval_args(context)
         if isinstance(value, Iterable) and not isinstance(value, (str, tuple)):
             store[self.handle].update(value)
         else:
             store[self.handle].add(value)
     def __call__(self, context: PostingContext) -> set:
         """Return the result for an aggregation."""
         return context.store[self.handle]  # type:ignore[no-any-return]
 class _EnvironmentMixin:
     db_path = Path('Financial', 'Ledger', 'supporters.db')
     columns: EnvironmentColumns
     functions: EnvironmentFunctions
     @classmethod
     def with_config(cls, config: configmod.Config) -> Type['_EnvironmentMixin']:
         columns = cls.columns.copy()
         repo_path = config.repository_path()
         try:
             if repo_path is None:
                 raise sqlite3.Error("no repository configured to host database")
             db_conn = sqlite3.connect(os.fspath(repo_path / cls.db_path))
         except (OSError, sqlite3.Error):
             columns['db_email'] = DBEmail
             columns['db_id'] = DBId
             columns['db_postal'] = DBPostal
         else:
             columns['db_email'] = DBEmail.with_db(db_conn)
             columns['db_id'] = DBId.with_db(db_conn)
             columns['db_postal'] = DBPostal.with_db(db_conn)
         rt_credentials = config.rt_credentials()
         rt_client = config.rt_client(rt_credentials)
         if rt_client is None:
             rt_ticket = RTTicket
         else:
             rt_ticket = RTTicket.with_client(rt_client, rt_credentials.idstr())
         functions = cls.functions.copy()
         functions[('rt_ticket', str, str)] = rt_ticket
         functions[('rt_ticket', str, str, int)] = rt_ticket
         return type(cls.__name__, (cls,), {
             'columns': columns,
             'functions': functions,
         })
 class FilterPostingsEnvironment(bc_query_env.FilterPostingsEnvironment, _EnvironmentMixin):
     columns: EnvironmentColumns  # type:ignore[assignment]
     functions: EnvironmentFunctions = bc_query_env.FilterPostingsEnvironment.functions.copy()  # type:ignore[assignment]
     functions['meta_docs'] = MetaDocs
     functions['str_meta'] = StrMeta
 class TargetsEnvironment(bc_query_env.TargetsEnvironment, _EnvironmentMixin):
     columns: EnvironmentColumns  # type:ignore[assignment]
     functions: EnvironmentFunctions = FilterPostingsEnvironment.functions.copy()  # type:ignore[assignment]
     functions.update(bc_query_env.AGGREGATOR_FUNCTIONS)
     functions['set'] = AggregateSet
 class BooksLoader:
     """Closure to load books with a zero-argument callable
     This matches the load interface that BQLShell expects.
     """
     def __init__(
             self,
             books_loader: Optional[books.Loader],
             start_date: Optional[datetime.date]=None,
             stop_date: Optional[datetime.date]=None,
             rewrite_rules: Sequence[rewrite.RewriteRuleset]=(),
     ) -> None:
         self.books_loader = books_loader
         self.start_date = start_date
         self.stop_date = stop_date
         self.rewrite_rules = rewrite_rules
     def __call__(self) -> books.LoadResult:
         logger.debug("BooksLoader called")
         result = books.Loader.dispatch(self.books_loader, self.start_date, self.stop_date)
         logger.debug("books loaded from Beancount")
         if self.rewrite_rules:
             for index, entry in enumerate(result.entries):
                 # entry might not be a Transaction; we catch that later.
                 # The type ignores are because the underlying Beancount type isn't
                 # type-checkable.
                 postings = data.Posting.from_txn(entry)  # type:ignore[arg-type]
                 for ruleset in self.rewrite_rules:
                     postings = ruleset.rewrite(postings)
                 try:
                     result.entries[index] = entry._replace(postings=list(postings))  # type:ignore[call-arg]
                 except AttributeError:
                     pass
             logger.debug("rewrite rules applied")
         return result
 class BQLShell(bc_query_shell.BQLShell):
     def __init__(
             self,
             config: configmod.Config,
             is_interactive: bool,
             loadfun: Callable[[], books.LoadResult],
             outfile: TextIO,
             default_format: str='text',
             do_numberify: bool=False,
     ) -> None:
         super().__init__(is_interactive, loadfun, outfile, default_format, do_numberify)
         rt_credentials = config.rt_credentials()
         rt_key = rt_credentials.idstr()
         rt_client = config.rt_client(rt_credentials)
         self.env_postings = FilterPostingsEnvironment.with_config(config)()
         self.env_targets = TargetsEnvironment.with_config(config)()
         self.ods = QueryODS(config.rt_wrapper(rt_credentials))
         self.last_line_parsed = ''
     def run_parser(
             self,
             line: str,
             default_close_date: Optional[datetime.datetime]=None,
     ) -> None:
         self.last_line_parsed = line
         super().run_parser(line, default_close_date)
     @functools.wraps(bc_query_shell.BQLShell.on_Select, ('__doc__',))
     def on_Select(self, statement: QueryStatement) -> None:
         output_format: str = self.vars['format']
         try:
             render_func = getattr(self, f'_render_{output_format}')
         except AttributeError:
             logger.error("unknown output format %r", output_format)
             return
         try:
             logger.debug("compiling query")
             compiled_query = bc_query_compile.compile(
                 statement, self.env_targets, self.env_postings, self.env_entries,
+            )
             logger.debug("executing query")
             row_types, rows = bc_query_execute.execute_query(
                 compiled_query, self.entries, self.options_map,
+            )
             if self.vars['numberify']:
                 logger.debug("numberifying query")
                 row_types, rows = bc_query_numberify.numberify_results(
                     row_types, rows, self.options_map['dcontext'].build(),
+                )
         except Exception as error:
             logger.error(str(error), exc_info=logger.isEnabledFor(logging.DEBUG))
             if (isinstance(error, TypeError)
                 and error.args
                 and ' not supported between instances ' in error.args[0]):
                 logger.info(
                     "HINT: Are you using ORDER BY or comparisons with metadata "
                     "that isn't consistently set?\n  "
                     "Try looking up that metadata with str_meta() instead to "
                     "ensure your comparisons use a consistent data type.",
+                )
             try:
                 hint_func = getattr(self, f'_hint_{type(error).__name__}')
             except AttributeError:
                 pass
             else:
                 hint_func(error, statement)
             return
         if not rows and output_format != 'ods':
             print("(empty)", file=self.outfile)
         else:
             logger.debug("rendering query as %s", output_format)
             render_func(statement, row_types, rows)
     def _hint_TypeError(self, error: TypeError, statement: QueryStatement) -> None:
         try:
             errmsg = str(error.args[0])
         except IndexError:
             return
         if ' not supported between instances ' in errmsg:
             logger.info(
                 "HINT: Are you using ORDER BY or comparisons with metadata "
                 "that isn't consistently set?\n  "
                 "Try looking up that metadata with str_meta() instead to "
                 "ensure your comparisons use a consistent data type.",
+            )
         elif errmsg.startswith('unhashable type: '):
             logger.info(
                 "HINT: bean-query does not support selecting columns or "
                 "functions that return multiple items as non-aggregate data in "
                 "GROUP BY queries.\n  "
                 "If you want to aggregate that data, run it through set().",
+            )
     def _render_csv(self, statement: QueryStatement, row_types: RowTypes, rows: Rows) -> None:
         bc_query_render.render_csv(
             row_types,
             rows,
             self.options_map['dcontext'],
             self.outfile,
             self.vars['expand'],
+        )
     def _render_ods(self, statement: QueryStatement, row_types: RowTypes, rows: Rows) -> None:
         self.ods.write_query(statement, row_types, rows, self.last_line_parsed)
         logger.info(
             "%s rows of results saved in sheet %s",
             len(rows),
             self.ods.sheet.getAttribute('name'),
+        )
     def _render_text(self, statement: QueryStatement, row_types: RowTypes, rows: Rows) -> None:
         with contextlib.ExitStack() as stack:
             if self.is_interactive:
                 output = stack.enter_context(self.get_pager())
             else:
                 output = self.outfile
             bc_query_render.render_text(
                 row_types,
                 rows,
                 self.options_map['dcontext'],
                 output,
                 self.vars['expand'],
                 self.vars['boxed'],
                 self.vars['spaced'],
+            )
 class QueryODS(core.BaseODS[NamedTuple, None]):
     META_FNAMES = frozenset([
         # Names of functions, as defined in Environments, that look up
         # posting metadata that could contain documentation links
         'any_meta',
         'entry_meta',
         'meta',
         'meta_docs',
         'str_meta',
     ])
     def is_empty(self) -> bool:
         return not self.sheet.childNodes
     def section_key(self, row: NamedTuple) -> None:
         return None
     def _generic_cell(self, value: Any) -> odf.table.TableCell:
         if isinstance(value, Iterable) and not isinstance(value, (str, tuple)):
             return self.multiline_cell(value)
         else:
             return self.string_cell('' if value is None else str(value))
     def _inventory_cell(self, value: Inventory) -> odf.table.TableCell:
         return self.balance_cell(core.Balance(pos.units for pos in value))
     def _link_string_cell(self, value: str) -> odf.table.TableCell:
         return self.meta_links_cell(value.split())
     def _metadata_cell(self, value: MetaValue) -> odf.table.TableCell:
         return self._cell_type(type(value))(value)
     def _position_cell(self, value: Position) -> odf.table.TableCell:
         return self.currency_cell(value.units)
     def _cell_type(self, row_type: Type) -> CellFunc:
         """Return a function to create a cell, for non-metadata row types."""
         if issubclass(row_type, Inventory):
             return self._inventory_cell
         elif issubclass(row_type, Position):
             return self._position_cell
         elif issubclass(row_type, BeancountAmount):
             return self.currency_cell
         elif issubclass(row_type, (int, float, Decimal)):
             return self.float_cell
         elif issubclass(row_type, datetime.date):
             return self.date_cell
         elif issubclass(row_type, str):
             return self.string_cell
         else:
             return self._generic_cell
     def _link_cell_type(self, row_type: Type) -> CellFunc:
         """Return a function to create a cell from metadata with documentation links."""
         if issubclass(row_type, str):
             return self._link_string_cell
         elif issubclass(row_type, tuple):
             return self._generic_cell
         elif issubclass(row_type, Iterable):
             return self.meta_links_cell
         else:
             return self._generic_cell
     def _meta_target(self, target: QueryExpression) -> Optional[MetaKey]:
         """Return the metadata key looked up by this target, if any
         This function takes a parsed target (i.e., what we're SELECTing) and
         recurses it to see whether it's looking up any metadata. If so, it
         returns the key of that metadata. Otherwise it returns None.
         """
         if isinstance(target, bc_query_parser.UnaryOp):
             return self._meta_target(target.operand)
         elif not isinstance(target, bc_query_parser.Function):
             return None
         try:
             operand = target.operands[0]
         except IndexError:
             return None
         if (target.fname in self.META_FNAMES
             and isinstance(operand, bc_query_parser.Constant)):
             return operand.value  # type:ignore[no-any-return]
         else:
             for operand in target.operands:
                 retval = self._meta_target(operand)
                 if retval is not None:
                     break
             return retval
     def _cell_types(self, statement: QueryStatement, row_types: RowTypes) -> Iterator[CellFunc]:
         """Return functions to create table cells from result rows
         Given a parsed query and the types of return rows, yields a function
         to create a cell for each column in the row, in order. The returned
         functions vary in order to provide the best available formatting for
         different data types.
         """
         if (isinstance(statement, bc_query_parser.Select)
             and isinstance(statement.targets, Sequence)):
             targets = [t.expression for t in statement.targets]
         else:
             # Synthesize something that makes clear we're not loading metadata.
             targets = [bc_query_parser.Column(name) for name, _ in row_types]
         for target, (_, row_type) in zip(targets, row_types):
             meta_key = self._meta_target(target)
             if meta_key is None:
                 yield self._cell_type(row_type)
             elif meta_key in data.LINK_METADATA:
                 yield self._link_cell_type(row_type)
             else:
                 yield self._metadata_cell
     def write_query(
             self,
             statement: QueryStatement,
             row_types: RowTypes,
             rows: Rows,
             query_string: Optional[str]=None,
     ) -> None:
         if self.is_empty():
             self.sheet.setAttribute('name', "Query 1")
         else:
             self.use_sheet(f"Query {len(self.document.spreadsheet.childNodes) + 1}")
         for name, row_type in row_types:
             if issubclass(row_type, datetime.date):
                 col_width = 1.0
             elif issubclass(row_type, (BeancountAmount, Inventory, Position)):
                 col_width = 1.5
             else:
                 col_width = 2.0
             col_style = self.column_style(col_width)
             self.sheet.addElement(odf.table.TableColumn(stylename=col_style))
         self.add_row(*(
             self.string_cell(data.Metadata.human_name(name), stylename=self.style_bold)
             for name, _ in row_types
         ))
         self.lock_first_row()
         if query_string:
             self.add_annotation(query_string, parent=self.sheet.lastChild.firstChild)
         cell_funcs = list(self._cell_types(statement, row_types))
         for row in rows:
             self.add_row(*(
                 cell_func(value)
                 for cell_func, value in zip(cell_funcs, row)
             ))
 class ReportFormat(enum.Enum):
     TEXT = 'text'
     TXT = TEXT
     CSV = 'csv'
     ODS = 'ods'
 class SetCYDates(argparse.Action):
     def __call__(self,
                  parser: argparse.ArgumentParser,
                  namespace: argparse.Namespace,
                  values: Union[Sequence[Any], str, None]=None,

setup.py

➞

Show inline comments

 #!/usr/bin/env python3
 from setuptools import setup
 setup(
     name='conservancy_beancount',
     description="Plugin, library, and reports for reading Conservancy's books",
-    version='1.19.3',
+    version='1.19.4',
     author='Software Freedom Conservancy',
     author_email='info@sfconservancy.org',
     license='GNU AGPLv3+',
     install_requires=[
         'babel>=2.6',  # Debian:python3-babel
         'beancount>=2.2',  # Debian:beancount
         'GitPython>=2.0',  # Debian:python3-git
         # 1.4.1 crashes when trying to save some documents.
         'odfpy>=1.4.0,!=1.4.1',  # Debian:python3-odf
         'pdfminer.six>=20200101',
         'python-dateutil>=2.7',  # Debian:python3-dateutil
         'PyYAML>=3.0',  # Debian:python3-yaml
         'regex',  # Debian:python3-regex
         'rt>=2.0',
     ],
     setup_requires=[
         'pytest-mypy',
         'pytest-runner',  # Debian:python3-pytest-runner
     ],
     tests_require=[
         'mypy>=0.770',  # Debian:python3-mypy
         'pytest',  # Debian:python3-pytest
     ],
     packages=[
         'conservancy_beancount',
         'conservancy_beancount.pdfforms',
         'conservancy_beancount.pdfforms.extract',
         'conservancy_beancount.plugin',
         'conservancy_beancount.reconcile',
         'conservancy_beancount.reports',
         'conservancy_beancount.tools',
     ],
     entry_points={
         'console_scripts': [
             'accrual-report = conservancy_beancount.reports.accrual:entry_point',
             'assemble-audit-reports = conservancy_beancount.tools.audit_report:entry_point',
             'balance-sheet-report = conservancy_beancount.reports.balance_sheet:entry_point',
             'budget-report = conservancy_beancount.reports.budget:entry_point',
             'bean-sort = conservancy_beancount.tools.sort_entries:entry_point',
             'extract-odf-links = conservancy_beancount.tools.extract_odf_links:entry_point',
             'fund-report = conservancy_beancount.reports.fund:entry_point',
             'ledger-report = conservancy_beancount.reports.ledger:entry_point',
             'opening-balances = conservancy_beancount.tools.opening_balances:entry_point',
             'pdfform-extract = conservancy_beancount.pdfforms.extract:entry_point',
             'pdfform-extract-irs990scheduleA = conservancy_beancount.pdfforms.extract.irs990scheduleA:entry_point',
             'pdfform-fill = conservancy_beancount.pdfforms.fill:entry_point',
             'query-report = conservancy_beancount.reports.query:entry_point',
             'reconcile-paypal = conservancy_beancount.reconcile.paypal:entry_point',
             'reconcile-statement = conservancy_beancount.reconcile.statement:entry_point',
             'split-ods-links = conservancy_beancount.tools.split_ods_links:entry_point',
         ],
     },
+)

0 comments (0 inline, 0 general)