for ttype, tvalue, _, _, _ in self.in_tokens:

            try:

                transformer = getattr(self, 'transform_' + tokenize.tok_name[ttype])

            except AttributeError:

                raise ValueError("{} token {!r} not supported".format(ttype, tvalue))

            yield from transformer(ttype, tvalue)

    def _noop_transformer(self, ttype, tvalue):

        yield (ttype, tvalue)

    transform_ENDMARKER = _noop_transformer

    transform_NEWLINE = _noop_transformer

    def transform_ENCODING(self, ttype, tvalue):

        self.in_encoding = tvalue

        return self._noop_transformer(ttype, tvalue)

    def transform(self):

        out_bytes = tokenize.untokenize(self)

        return out_bytes.decode(self.in_encoding)

class AmountTokenTransformer(TokenTransformer):

    SUPPORTED_NAMES = frozenset([

        'if',

        'else',

        'and',

        'or',

        'not',

        'in',

    ])

    SUPPORTED_OPS = frozenset([

        '(',

        ')',

        '+',

        '-',

        '*',

        '/',

        '==',

        '!=',

        '<',

        '<=',

        '>',

        '>=',

    ])

    def __iter__(self):

        tokens = super().__iter__()

        for token in tokens:

            yield token

            if token[0] == tokenize.NAME:

                break

        else:

            raise ValueError("no amount in expression")

        yield from tokens

    def transform_NAME(self, ttype, tvalue):

        if tvalue in self.SUPPORTED_NAMES:

            yield from self._noop_transformer(ttype, tvalue)

        else:

            raise ValueError("unsupported bare word {!r}".format(tvalue))

    def transform_NUMBER(self, ttype, tvalue):

        yield (tokenize.NAME, 'Decimal')

        yield (tokenize.OP, '(')

        yield (tokenize.STRING, repr(tvalue))

        yield (tokenize.OP, ')')

    def transform_OP(self, ttype, tvalue):

        if tvalue == '{':

            try:

                name_type, name_value, _, _, _ = next(self.in_tokens)

                close_type, close_value, _, _, _ = next(self.in_tokens)

                if (name_type != tokenize.NAME

                    or name_value != name_value.lower()

                    or close_type != tokenize.OP

                    or close_value != '}'):

                    raise ValueError()

            except (StopIteration, ValueError):

                raise ValueError("opening { does not name variable")

            yield (tokenize.NAME, name_value)

        elif tvalue in self.SUPPORTED_OPS:

            yield from self._noop_transformer(ttype, tvalue)

        else:

            raise ValueError("unsupported operator {!r}".format(tvalue))

    transform_STRING = TokenTransformer._noop_transformer

class AccountSplitter:

    EVAL_GLOBALS = {

        'Decimal': decimal.Decimal,

    }

    TARGET_LINE_LEN = 78

    # -4 because that's how many spaces prefix an account line.

    TARGET_ACCTLINE_LEN = TARGET_LINE_LEN - 4

    def __init__(self, signed_currencies, signed_currency_fmt, unsigned_currency_fmt,

                 template_name):

        self.splits = []

        self.metadata = []

        self.signed_currency_fmt = signed_currency_fmt

        self.unsigned_currency_fmt = unsigned_currency_fmt

        self.signed_currencies = set(signed_currencies)

        self.template_name = template_name

        self._last_template_vars = object()

    def is_empty(self):

        return not self.splits

    def add(self, account, amount_expr):

        try:

            clean_expr = AmountTokenTransformer.from_str(amount_expr).transform()

            compiled_expr = compile(clean_expr, self.template_name, 'eval')

        except (SyntaxError, tokenize.TokenError, ValueError) as error:

            raise errors.UserInputConfigurationError(error.args[0], amount_expr)

        else:

            self.splits.append((account, compiled_expr))

            self.metadata.append('')

    def set_metadata(self, metadata_s):

        self.metadata[-1] = metadata_s

    def _currency_decimal(self, amount, currency):

        return decimal.Decimal(babel.numbers.format_currency(amount, currency, '###0.###'))

    def _balance_amounts(self, amounts, to_amount):

        cmp_func = operator.lt if to_amount > 0 else operator.gt

        should_balance = functools.partial(cmp_func, 0)

        remainder = to_amount

        balance_index = None

        for index, (_, amount) in enumerate(amounts):

            if should_balance(amount):

                remainder -= amount

                balance_index = index

        if balance_index is None:

            pass

        elif (abs(remainder) / abs(to_amount)) >= decimal.Decimal('.1'):

            raise errors.UserInputConfigurationError(

                "template can't balance amounts to {}".format(to_amount),

                self.template_name,

            )

        else:

            account_name, start_amount = amounts[balance_index]

            amounts[balance_index] = (account_name, start_amount + remainder)

    def _build_amounts(self, template_vars):

        try:

            amounts = [

                (account,

                 self._currency_decimal(eval(amount_expr, self.EVAL_GLOBALS, template_vars),

                                        template_vars['currency']),

                ) for account, amount_expr in self.splits

            ]

        except (ArithmeticError, NameError, TypeError, ValueError) as error:

            raise errors.UserInputConfigurationError(

                "{}: {}".format(type(error).__name__, error),

                "template {!r}".format(self.template_name)

            ) from error

        if sum(amt for _, amt in amounts) != 0:

            self._balance_amounts(amounts, template_vars['amount'])

            self._balance_amounts(amounts, -template_vars['amount'])

        return amounts

    def _iter_splits(self, template_vars):

        amounts = self._build_amounts(template_vars)

        if template_vars['currency'] in self.signed_currencies:

            amt_fmt = self.signed_currency_fmt

        else:

            amt_fmt = self.unsigned_currency_fmt

        for (account, amount), metadata in zip(amounts, self.metadata):

            if amount == 0:

                yield ''

            else:

                account_s = account.format_map(template_vars)

                amount_s = babel.numbers.format_currency(amount, template_vars['currency'], amt_fmt)

                sep_len = max(2, self.TARGET_ACCTLINE_LEN - len(account_s) - len(amount_s))

                yield '\n    {}{}{}{}'.format(

                    account_s, ' ' * sep_len, amount_s,

                    metadata.format_map(template_vars),

                )

    def render_next(self, template_vars):

        if template_vars is not self._last_template_vars:

            self._split_iter = self._iter_splits(template_vars)

            self._last_template_vars = template_vars

        return next(self._split_iter)

class Template:

    ACCOUNT_SPLIT_RE = re.compile(r'(?:\t|  )\s*')

    DATE_FMT = '%Y/%m/%d'

    SIGNED_CURRENCY_FMT = '¤#,##0.###;¤-#,##0.###'

    UNSIGNED_CURRENCY_FMT = '#,##0.### ¤¤'

    def __init__(self, template_s, signed_currencies=frozenset(),

                 date_fmt=DATE_FMT,

                 signed_currency_fmt=SIGNED_CURRENCY_FMT,

                 unsigned_currency_fmt=UNSIGNED_CURRENCY_FMT,

                 template_name='<template>'):

        self.date_fmt = date_fmt

        self.splitter = AccountSplitter(

            signed_currencies, signed_currency_fmt, unsigned_currency_fmt, template_name)

        lines = self._template_lines(template_s)

        self.format_funcs = []

        try:

            self.format_funcs.append(next(lines).format_map)

        except StopIteration:

            return

        metadata = []

        for line in lines:

            if line.startswith(';'):

                metadata.append(line)

            else:

                self._add_str_func(metadata)

                metadata = []

                line = line.strip()

                match = self.ACCOUNT_SPLIT_RE.search(line)

                if match is None:

                    raise errors.UserInputError("no amount expression found", line)

                account = line[:match.start()]

                amount_expr = line[match.end():]

                self.splitter.add(account, amount_expr)

                self.format_funcs.append(self.splitter.render_next)

        self._add_str_func(metadata)

        self.format_funcs.append('\n'.format_map)

    def _nonblank_lines(self, s):

        for line in s.splitlines(True):

            line = line.strip()

            if line:

                yield line

    def _template_lines(self, template_s):

        lines = self._nonblank_lines(template_s)

        try:

            line1 = next(lines)

        except StopIteration:

            return

            yield '\n' + line1

        else:

            yield '\n{date} {payee}'

            yield line1

        yield from lines

    def _add_str_func(self, str_seq):

        str_flat = ''.join('\n      ' + s for s in str_seq)

        if not str_flat:

            pass

        elif self.splitter.is_empty():

            self.format_funcs.append(str_flat.format_map)

        else:

            self.splitter.set_metadata(str_flat)

    def render(self, template_vars):

        # template_vars must have these keys.  Raise a KeyError if not.

        template_vars['currency']

        template_vars['payee']

        render_vars = {

            'amount': strparse.currency_decimal(template_vars['amount']),

        }

        for key, value in template_vars.items():

                render_vars[key] = value.strftime(self.date_fmt)

        all_vars = collections.ChainMap(render_vars, template_vars)

        return ''.join(f(all_vars) for f in self.format_funcs)

    def is_empty(self):

        return not self.format_funcs

class LedgerEntryHook:

    KIND = HOOK_KINDS.OUTPUT

    def __init__(self, config):

        self.config = config

    @staticmethod

    @functools.lru_cache()

    def _load_template(config, section_name, config_key):

        section_config = config.get_section(section_name)

        try:

            template_s = section_config[config_key]

        except KeyError:

            raise errors.UserInputConfigurationError(

                "Ledger template not defined in [{}]".format(section_name),

                config_key,

            )

        return Template(

            template_s,

            date_fmt=section_config['date_format'],

            signed_currencies=[code.strip().upper() for code in section_config['signed_currencies'].split(',')],

            signed_currency_fmt=section_config['signed_currency_format'],

            unsigned_currency_fmt=section_config['unsigned_currency_format'],

            template_name=config_key,

        )

    def run(self, entry_data):

        try:

            template_key = entry_data['ledger template']

        except KeyError:

            template_key = '{} {} ledger entry'.format(

                strparse.rslice_words(entry_data['importer_module'], -1, '.', 1),

                entry_data['importer_class'][:-8].lower(),

            )

        try:

            template = self._load_template(self.config, None, template_key)

        except errors.UserInputConfigurationError as error:

            if error.strerror.startswith('Ledger template not defined '):

                have_template = False

            else:

                raise

        else:

            have_template = not template.is_empty()

        if not have_template:

            logger.warning("no Ledger template defined as %r", template_key)

        else:

            with self.config.open_output_file() as out_file:

                print(template.render(entry_data), file=out_file, end='')

#!/usr/bin/env python3

import sys

from setuptools import setup, find_packages

REQUIREMENTS = {

    'install_requires': [

        'babel',

        'enum34;python_version<"3.4"',

    ],

    'setup_requires': ['pytest-runner'],

    'extras_require': {

        'brightfunds': ['xlrd'],

        'nbpy2017': ['beautifulsoup4', 'html5lib'],

    },

}

all_extras_require = [

    req for reqlist in REQUIREMENTS['extras_require'].values() for req in reqlist

]

REQUIREMENTS['extras_require']['all_importers'] = all_extras_require

REQUIREMENTS['tests_require'] = [

    'pytest',

    'PyYAML',

    *all_extras_require,

]

setup(

    name='import2ledger',

    description="Import different sources of financial data to Ledger",

    author='Brett Smith',

    author_email='brettcsmith@brettcsmith.org',

    license='GNU AGPLv3+',

    packages=find_packages(include=['import2ledger', 'import2ledger.*']),

    entry_points={

        'console_scripts': ['import2ledger = import2ledger.__main__:main'],

    },

    **REQUIREMENTS,

)

import collections

import configparser

import contextlib

import datetime

import decimal

import io

import pathlib

import pytest

from import2ledger import errors

from import2ledger.hooks import ledger_entry

from . import DATA_DIR, normalize_whitespace

DATE = datetime.date(2015, 3, 14)

config = configparser.ConfigParser(comment_prefixes='#')

with pathlib.Path(DATA_DIR, 'templates.ini').open() as conffile:

    config.read_file(conffile)

def template_from(section_name, *args, **kwargs):

    return ledger_entry.Template(config[section_name]['template'], *args, **kwargs)

def template_vars(payee, amount, currency='USD', date=DATE, other_vars=None):

    call_vars = {

        'amount': decimal.Decimal(amount),

        'currency': currency,

        'date': date,

        'payee': payee,

        'ledger template': 'template',

    }

    if other_vars is None:

        return call_vars

    else:

        return collections.ChainMap(call_vars, other_vars)

def render_lines(render_vars, section_name, *args, **kwargs):

    tmpl = template_from(section_name, *args, **kwargs)

    rendered = tmpl.render(render_vars)

    return [normalize_whitespace(s) for s in rendered.splitlines()]

def assert_easy_render(tmpl, entity, amount, currency, expect_date, expect_amt):

    rendered = tmpl.render(template_vars(entity, amount, currency))

    lines = [normalize_whitespace(s) for s in rendered.splitlines()]

    assert lines == [

        "",

        "{} {}".format(expect_date, entity),

        "  Accrued:Accounts Receivable  " + expect_amt,

        "  Income:Donations  " + expect_amt.replace(amount, "-" + amount),

    ]

    assert not tmpl.is_empty()

def test_easy_template():

    tmpl = template_from('Simplest')

    assert_easy_render(tmpl, 'JJ', '5.99', 'CAD', '2015/03/14', '5.99 CAD')

def test_date_formatting():

    tmpl = template_from('Simplest', date_fmt='%Y-%m-%d')

    assert_easy_render(tmpl, 'KK', '6.99', 'CAD', '2015-03-14', '6.99 CAD')

def test_currency_formatting():

    tmpl = template_from('Simplest', signed_currencies=['USD'])

    assert_easy_render(tmpl, 'CC', '7.99', 'USD', '2015/03/14', '$7.99')

def test_empty_template():

    tmpl = ledger_entry.Template("\n \n")

    assert tmpl.render(template_vars('BB', '8.99')) == ''

    assert tmpl.is_empty()

def test_complex_template():

    render_vars = template_vars('TT', '125.50', other_vars={

        'entity': 'T-T',

        'program': 'Spectrum Defense',

        'txid': 'ABCDEF',

    })

    lines = render_lines(

        render_vars, 'Complex',

        date_fmt='%Y-%m-%d',

        signed_currencies=['USD'],

    )

    assert lines == [

        "",

        "2015-03-14 TT",

        "  ;Tag: Value",

        "  ;TransactionID: ABCDEF",

        "  Accrued:Accounts Receivable  $125.50",

        "  ;Entity: Supplier",

        "  Income:Donations:Spectrum Defense  $-119.85",

        "  ;Program: Spectrum Defense",

        "  ;Entity: T-T",

        "  Income:Donations:General  $-5.65",

        "  ;Entity: T-T",

    ]

def test_balancing():

    lines = render_lines(template_vars('FF', '1.01'), 'FiftyFifty')

    assert lines == [

        "",

        "2015/03/14 FF",

        "  Accrued:Accounts Receivable  1.01 USD",

        "  Income:Donations  -0.50 USD",

        "  Income:Sales  -0.51 USD",

    ]

def test_multivalue():

    render_vars = template_vars('DD', '150.00', other_vars={

        'tax': decimal.Decimal('12.50'),

    })

    lines = render_lines(render_vars, 'Multivalue')

    assert lines == [

        "",

        "2015/03/14 DD",

        "  Expenses:Taxes  12.50 USD",

        "  ;TaxAuthority: IRS",

        "  Accrued:Accounts Receivable  137.50 USD",

        "  Income:RBI  -15.00 USD",

        "  Income:Donations  -135.00 USD",

    ]

def test_zeroed_account_skipped():

    render_vars = template_vars('GG', '110.00', other_vars={

        'tax': decimal.Decimal(0),

    })

    lines = render_lines(render_vars, 'Multivalue')

    assert lines == [

        "",

        "2015/03/14 GG",

        "  Accrued:Accounts Receivable  110.00 USD",

        "  Income:RBI  -11.00 USD",

        "  Income:Donations  -99.00 USD",

    ]

def test_zeroed_account_last():

    render_vars = template_vars('JJ', '90.00', other_vars={

        'item_sales': decimal.Decimal(0),

    })

    lines = render_lines(render_vars, 'Multisplit')

    assert lines == [

        "",

        "2015/03/14 JJ",

        "  Assets:Cash  90.00 USD",

        "  Income:Sales  -90.00 USD",

        "  ; :NonItem:",

    ]

def test_multiple_postings_same_account():

    render_vars = template_vars('LL', '80.00', other_vars={

        'item_sales': decimal.Decimal(30),

    })

    lines = render_lines(render_vars, 'Multisplit')

    assert lines == [

        "",

        "2015/03/14 LL",

        "  Assets:Cash  80.00 USD",

        "  Income:Sales  -50.00 USD",

        "  ; :NonItem:",

        "  Income:Sales  -30.00 USD",

        "  ; :Item:",

    ]

def test_custom_payee_line():

    render_vars = template_vars('ZZ', '10.00', other_vars={

        'custom_date': datetime.date(2014, 2, 13),

    })

    lines = render_lines(render_vars, 'Custom Payee')

    assert lines == [

        "",

        "2014/02/13  ZZ - Custom",

        "  Accrued:Accounts Receivable  10.00 USD",

        "  Income:Donations  -10.00 USD",

    ]

def test_line1_not_custom_payee():

    render_vars = template_vars('VV', '15.00', other_vars={

        'custom_date': datetime.date(2014, 2, 12),

    })

    lines = render_lines(render_vars, 'Simplest')

    assert lines == [

        "",

        "2015/03/14 VV",

        "  Accrued:Accounts Receivable  15.00 USD",

        "  Income:Donations  -15.00 USD",

    ]

@pytest.mark.parametrize('amount,expect_fee', [

    (40, 3),

    (80, 6),

])

def test_conditional(amount, expect_fee):

    expect_cash = amount - expect_fee

    amount_s = '{:.02f}'.format(amount)

    render_vars = template_vars('Buyer', amount_s)

    lines = render_lines(render_vars, 'Conditional')

    assert lines == [

        "",

        "2015/03/14 Buyer",

        "  Assets:Cash  {:.02f} USD".format(expect_cash),

        "  Expenses:Banking Fees  {:.02f} USD".format(expect_fee),

        "  Income:Sales  -{} USD".format(amount_s),

    ]

def test_string_conditionals():

    render_vars = template_vars('MM', '6', other_vars={

        'false': '',

        'true': 'true',

    })

    lines = render_lines(render_vars, 'StringConditional')

    assert lines == [

        "",

        "2015/03/14 MM",

        "  Income:Sales  -1.00 USD",

        "  Income:Sales  -2.00 USD",

        "  Income:Sales  -3.00 USD",

        "  Assets:Cash  6.00 USD",

    ]

def test_self_balanced():

    # The amount needs to be different than what's in the template.

    render_vars = template_vars('NN', '0')

    lines = render_lines(render_vars, 'SelfBalanced')

    assert lines == [

        "",

        "2015/03/14 NN",

        "  Income:Sales  -5.00 USD",

        "  Assets:Cash  5.00 USD",

    ]

@pytest.mark.parametrize('amount_expr', [

    '',

    'name',

    '-',

    '()',

    '+()',

    '{}',

    '{{}}',

    '{()}',

    '{name',

    'name}',

    '{42}',

    '(5).real',

    '{amount.real}',

    '{amount.is_nan()}',

    '{Decimal}',

    '{FOO}',

])

def test_bad_amount_expression(amount_expr):

    with pytest.raises(errors.UserInputError):

        ledger_entry.Template(" Income  " + amount_expr)

class Config:

    def __init__(self, use_section):

        self.section_name = use_section

        self.stdout = io.StringIO()

    @contextlib.contextmanager

    def open_output_file(self):

        yield self.stdout

    def get_section(self, name=None):

        return config[self.section_name]

def run_hook(entry_data, config_section):

    hook_config = Config(config_section)

    hook = ledger_entry.LedgerEntryHook(hook_config)

    assert hook.run(entry_data) is None

    stdout = hook_config.stdout.getvalue()

    return normalize_whitespace(stdout).splitlines()

def test_hook_renders_template():

    entry_data = template_vars('BB', '0.99')

    lines = run_hook(entry_data, 'Simplest')

    assert lines == [

        "",

        "2015-03-14 BB",

        "  Accrued:Accounts Receivable  $0.99",

        "  Income:Donations  -$0.99",

    ]

def test_hook_handles_empty_template():

    entry_data = template_vars('CC', 1)

    assert not run_hook(entry_data, 'Empty')

def test_hook_handles_template_undefined():

    entry_data = template_vars('DD', 1)

    assert not run_hook(entry_data, 'Nonexistent')

def test_string_value_is_user_error():

    entry_data = template_vars('EE', 1)

    with pytest.raises(errors.UserInputConfigurationError):

        run_hook(entry_data, 'NondecimalWord')

def test_string_variable_is_user_error():

    entry_data = template_vars('FF', 1)

    with pytest.raises(errors.UserInputConfigurationError):

        run_hook(entry_data, 'NondecimalVariable')