From 89378cbf90fc69b6df9d1dd1b8c5e7134a3e9b2f 2020-01-10 18:27:34
From: Brett Smith <brettcsmith@brettcsmith.org>
Date: 2020-01-10 18:27:34
Subject: [PATCH] hooks.ledger_entry: Clean up whitespace in strings.

The hook now ensures it does not output whitespace that could be
significant to Ledger, either because it's a newline or an
account-amount separator.

---

diff --git a/import2ledger/hooks/ledger_entry.py b/import2ledger/hooks/ledger_entry.py
index a677dcf30a9df45493513b7944e720521ac40948..805dc8a7c0fb0b7503a5fec99b9bcb786bce3168 100644
--- a/import2ledger/hooks/ledger_entry.py
+++ b/import2ledger/hooks/ledger_entry.py
@@ -222,9 +222,10 @@ class AccountSplitter:
 
 
 class Template:
-    ACCOUNT_SPLIT_RE = re.compile(r'(?:\t|  )\s*')
+    ACCOUNT_SPLIT_RE = re.compile(r'(?: ?\t|  )[ \t]*')
     DATE_FMT = '%Y/%m/%d'
     PAYEE_LINE_RE = re.compile(r'^\{(\w*_)*date\}\s')
+    NEWLINE_RE = re.compile(r'[\f\n\r\v\u0085\u2028\u2029]')
     SIGNED_CURRENCY_FMT = '¤#,##0.###;¤-#,##0.###'
     UNSIGNED_CURRENCY_FMT = '#,##0.### ¤¤'
 
@@ -301,12 +302,17 @@ class Template:
                 "entry needs {} field but that's not set by the importer".format(
                     self.date_field,
                 ), self.splitter.template_name)
-        render_vars = {
-            'amount': strparse.currency_decimal(template_vars['amount']),
-        }
+        render_vars = {}
         for key, value in template_vars.items():
-            if value is not None and (key == 'date' or key.endswith('_date')):
+            if value is None:
+                pass
+            elif key == 'date' or key.endswith('_date'):
                 render_vars[key] = value.strftime(self.date_fmt)
+            elif isinstance(value, str):
+                value = self.NEWLINE_RE.sub(' ', value)
+                value = self.ACCOUNT_SPLIT_RE.sub(' ', value)
+                render_vars[key] = value
+        render_vars['amount'] = strparse.currency_decimal(template_vars['amount'])
         all_vars = collections.ChainMap(render_vars, template_vars)
         return ''.join(f(all_vars) for f in self.format_funcs)
 
diff --git a/setup.py b/setup.py
index aaf15b543d673f716de7ba45a30b57850491fb83..8de41b7e7f1e244835a9e2d50cbdcacf699e5da1 100755
--- a/setup.py
+++ b/setup.py
@@ -30,7 +30,7 @@ REQUIREMENTS['tests_require'] = [
 setup(
     name='import2ledger',
     description="Import different sources of financial data to Ledger",
-    version='0.11.0',
+    version='0.11.1',
     author='Brett Smith',
     author_email='brettcsmith@brettcsmith.org',
     license='GNU AGPLv3+',
diff --git a/tests/test_hook_ledger_entry.py b/tests/test_hook_ledger_entry.py
index e2883324181f15b5468f79141bc6b8842daf7343..4b07ae8ece0f7da9e653de468933159017e66691 100644
--- a/tests/test_hook_ledger_entry.py
+++ b/tests/test_hook_ledger_entry.py
@@ -92,6 +92,38 @@ def test_complex_template():
         "  ;Entity: T-T",
     ]
 
+def test_variable_whitespace_cleaned():
+    # There are two critical parts of this to avoid making malformed Ledger
+    # entries:
+    # * Ensure newlines become non-newline whitespace so we don't write
+    #   malformed lines.
+    # * Collapse multiple spaces into one so variables in account names
+    #   can't make malformed account lines by having a premature split
+    #   between the account name and amount.
+    render_vars = template_vars('W\t\tS', '125.50', other_vars={
+        'entity': 'W\fS',
+        'program': 'Spectrum\r\nDefense',
+        'txid': 'ABC\v\tDEF',
+    })
+    lines = render_lines(
+        render_vars, 'Complex',
+        date_fmt='%Y-%m-%d',
+        signed_currencies=['USD'],
+    )
+    assert lines == [
+        "",
+        "2015-03-14 W S",
+        "  ;Tag: Value",
+        "  ;TransactionID: ABC DEF",
+        "  Accrued:Accounts Receivable  $125.50",
+        "  ;Entity: Supplier",
+        "  Income:Donations:Spectrum Defense  $-119.85",
+        "  ;Program: Spectrum Defense",
+        "  ;Entity: W S",
+        "  Income:Donations:General  $-5.65",
+        "  ;Entity: W S",
+    ]
+
 def test_balancing():
     lines = render_lines(template_vars('FF', '1.01'), 'FiftyFifty')
     assert lines == [