Changeset - 53413388e016
[Not reviewed]
0 7 0
Christopher Neugebauer - 8 years ago 2016-04-06 12:59:00
chrisjrn@gmail.com
Optimises queries through simplifying repeated queries and select_related use
7 files changed with 95 insertions and 38 deletions:
0 comments (0 inline, 0 general)
registrasion/controllers/cart.py
Show inline comments
...
 
@@ -59,48 +59,53 @@ class CartController(object):
 

	
 
        self.cart.time_last_updated = timezone.now()
 
        self.cart.reservation_duration = max(reservations)
 

	
 
    def end_batch(self):
 
        ''' Performs operations that occur occur at the end of a batch of
 
        product changes/voucher applications etc.
 
        THIS SHOULD BE PRIVATE
 
        '''
 

	
 
        self.recalculate_discounts()
 

	
 
        self.extend_reservation()
 
        self.cart.revision += 1
 
        self.cart.save()
 

	
 
    @transaction.atomic
 
    def set_quantities(self, product_quantities):
 
        ''' Sets the quantities on each of the products on each of the
 
        products specified. Raises an exception (ValidationError) if a limit
 
        is violated. `product_quantities` is an iterable of (product, quantity)
 
        pairs. '''
 

	
 
        items_in_cart = rego.ProductItem.objects.filter(cart=self.cart)
 
        items_in_cart = items_in_cart.select_related(
 
            "product",
 
            "product__category",
 
        )
 

	
 
        product_quantities = list(product_quantities)
 

	
 
        # n.b need to add have the existing items first so that the new
 
        # items override the old ones.
 
        all_product_quantities = dict(itertools.chain(
 
            ((i.product, i.quantity) for i in items_in_cart.all()),
 
            product_quantities,
 
        )).items()
 

	
 
        # Validate that the limits we're adding are OK
 
        self._test_limits(all_product_quantities)
 

	
 
        for product, quantity in product_quantities:
 
            try:
 
                product_item = rego.ProductItem.objects.get(
 
                    cart=self.cart,
 
                    product=product,
 
                )
 
                product_item.quantity = quantity
 
                product_item.save()
 
            except ObjectDoesNotExist:
 
                rego.ProductItem.objects.create(
 
                    cart=self.cart,
 
                    product=product,
...
 
@@ -262,75 +267,79 @@ class CartController(object):
 
        if errors:
 
            raise ValidationError(errors)
 

	
 
    @transaction.atomic
 
    def fix_simple_errors(self):
 
        ''' This attempts to fix the easy errors raised by ValidationError.
 
        This includes removing items from the cart that are no longer
 
        available, recalculating all of the discounts, and removing voucher
 
        codes that are no longer available. '''
 

	
 
        # Fix vouchers first (this affects available discounts)
 
        active_carts = rego.Cart.reserved_carts()
 
        to_remove = []
 
        for voucher in self.cart.vouchers.all():
 
            try:
 
                self._test_voucher(voucher)
 
            except ValidationError as ve:
 
                to_remove.append(voucher)
 

	
 
        for voucher in to_remove:
 
            self.cart.vouchers.remove(voucher)
 

	
 
        # Fix products and discounts
 
        items = rego.ProductItem.objects.filter(cart=self.cart)
 
        items = items.select_related("product")
 
        products = set(i.product for i in items)
 
        available = set(ProductController.available_products(
 
            self.cart.user,
 
            products=products,
 
        ))
 

	
 
        not_available = products - available
 
        zeros = [(product, 0) for product in not_available]
 

	
 
        self.set_quantities(zeros)
 

	
 
    @transaction.atomic
 
    def recalculate_discounts(self):
 
        ''' Calculates all of the discounts available for this product.
 
        '''
 

	
 
        # Delete the existing entries.
 
        rego.DiscountItem.objects.filter(cart=self.cart).delete()
 

	
 
        product_items = self.cart.productitem_set.all()
 
        product_items = self.cart.productitem_set.all().select_related(
 
            "product", "product__category",
 
        )
 

	
 
        products = [i.product for i in product_items]
 
        discounts = discount.available_discounts(self.cart.user, [], products)
 

	
 
        # The highest-value discounts will apply to the highest-value
 
        # products first.
 
        product_items = self.cart.productitem_set.all()
 
        product_items = product_items.select_related("product")
 
        product_items = product_items.order_by('product__price')
 
        product_items = reversed(product_items)
 
        for item in product_items:
 
            self._add_discount(item.product, item.quantity, discounts)
 

	
 
    def _add_discount(self, product, quantity, discounts):
 
        ''' Applies the best discounts on the given product, from the given
 
        discounts.'''
 

	
 
        def matches(discount):
 
            ''' Returns True if and only if the given discount apples to
 
            our product. '''
 
            if isinstance(discount.clause, rego.DiscountForCategory):
 
                return discount.clause.category == product.category
 
            else:
 
                return discount.clause.product == product
 

	
 
        def value(discount):
 
            ''' Returns the value of this discount clause
 
            as applied to this product '''
 
            if discount.clause.percentage is not None:
 
                return discount.clause.percentage * product.price
 
            else:
 
                return discount.clause.price
registrasion/controllers/category.py
Show inline comments
 
from registrasion import models as rego
 

	
 
from django.db.models import Sum
 

	
 

	
 
class AllProducts(object):
 
    pass
 

	
 

	
 
class CategoryController(object):
 

	
 
    def __init__(self, category):
 
        self.category = category
 

	
 
    @classmethod
 
    def available_categories(cls, user, products=AllProducts):
 
        ''' Returns the categories available to the user. Specify `products` if
 
        you want to restrict to just the categories that hold the specified
 
        products, otherwise it'll do all. '''
 

	
 
        # STOPGAP -- this needs to be elsewhere tbqh
 
        from product import ProductController
 

	
 
        if products is AllProducts:
 
            products = rego.Product.objects.all()
 
            products = rego.Product.objects.all().select_related("category")
 

	
 
        available = ProductController.available_products(
 
            user,
 
            products=products,
 
        )
 

	
 
        return set(i.category for i in available)
 

	
 
    def user_quantity_remaining(self, user):
 
        ''' Returns the number of items from this category that the user may
 
        add in the current cart. '''
 

	
 
        cat_limit = self.category.limit_per_user
 

	
 
        if cat_limit is None:
 
            # We don't need to waste the following queries
 
            return 99999999
 

	
 
        carts = rego.Cart.objects.filter(
 
            user=user,
 
            active=False,
 
            released=False,
 
        )
 

	
registrasion/controllers/conditions.py
Show inline comments
 
import itertools
 
import operator
 

	
 
from collections import defaultdict
 
from collections import namedtuple
 

	
 
from django.db.models import Sum
 
from django.utils import timezone
 

	
 
from registrasion import models as rego
 

	
 

	
 
ConditionAndRemainder = namedtuple(
 
    "ConditionAndRemainder",
 
    (
 
        "condition",
 
        "remainder",
 
    ),
 
)
 

	
 

	
 
class ConditionController(object):
 
    ''' Base class for testing conditions that activate EnablingCondition
 
    or Discount objects. '''
 

	
 
    def __init__(self):
...
 
@@ -69,77 +70,91 @@ class ConditionController(object):
 
            cls, user, products=None, product_quantities=None):
 
        ''' Evaluates all of the enabling conditions on the given products.
 

	
 
        If `product_quantities` is supplied, the condition is only met if it
 
        will permit the sum of the product quantities for all of the products
 
        it covers. Otherwise, it will be met if at least one item can be
 
        accepted.
 

	
 
        If all enabling conditions pass, an empty list is returned, otherwise
 
        a list is returned containing all of the products that are *not
 
        enabled*. '''
 

	
 
        if products is not None and product_quantities is not None:
 
            raise ValueError("Please specify only products or "
 
                             "product_quantities")
 
        elif products is None:
 
            products = set(i[0] for i in product_quantities)
 
            quantities = dict((product, quantity)
 
                              for product, quantity in product_quantities)
 
        elif product_quantities is None:
 
            products = set(products)
 
            quantities = {}
 

	
 
        # Get the conditions covered by the products themselves
 
        all_conditions = [
 
            product.enablingconditionbase_set.select_subclasses() |
 
            product.category.enablingconditionbase_set.select_subclasses()
 

	
 
        prods = (
 
            product.enablingconditionbase_set.select_subclasses()
 
            for product in products
 
        ]
 
        all_conditions = set(itertools.chain(*all_conditions))
 
        )
 
        # Get the conditions covered by their categories
 
        cats = (
 
            category.enablingconditionbase_set.select_subclasses()
 
            for category in set(product.category for product in products)
 
        )
 

	
 
        if products:
 
            # Simplify the query.
 
            all_conditions = reduce(operator.or_, itertools.chain(prods, cats))
 
        else:
 
            all_conditions = []
 

	
 
        # All mandatory conditions on a product need to be met
 
        mandatory = defaultdict(lambda: True)
 
        # At least one non-mandatory condition on a product must be met
 
        # if there are no mandatory conditions
 
        non_mandatory = defaultdict(lambda: False)
 

	
 
        messages = {}
 

	
 
        for condition in all_conditions:
 
            cond = cls.for_condition(condition)
 
            remainder = cond.user_quantity_remaining(user)
 

	
 
            # Get all products covered by this condition, and the products
 
            # from the categories covered by this condition
 
            cond_products = condition.products.all()
 
            from_category = rego.Product.objects.filter(
 
                category__in=condition.categories.all(),
 
            ).all()
 
            all_products = set(itertools.chain(cond_products, from_category))
 

	
 
            all_products = cond_products | from_category
 
            all_products = all_products.select_related("category")
 
            # Remove the products that we aren't asking about
 
            all_products = all_products & products
 
            all_products = [
 
                product
 
                for product in all_products
 
                if product in products
 
            ]
 

	
 
            if quantities:
 
                consumed = sum(quantities[i] for i in all_products)
 
            else:
 
                consumed = 1
 
            met = consumed <= remainder
 

	
 
            if not met:
 
                items = ", ".join(str(product) for product in all_products)
 
                base = cls.MESSAGE[remainder == 0][len(all_products) == 1]
 
                message = base % {"items": items, "remainder": remainder}
 

	
 
            for product in all_products:
 
                if condition.mandatory:
 
                    mandatory[product] &= met
 
                else:
 
                    non_mandatory[product] |= met
 

	
 
                if not met and product not in messages:
 
                    messages[product] = message
 

	
 

	
 
        valid = defaultdict(lambda: True)
 
        for product in itertools.chain(mandatory, non_mandatory):
registrasion/controllers/discount.py
Show inline comments
 
import itertools
 

	
 
from conditions import ConditionController
 
from registrasion import models as rego
 

	
 
from django.db.models import Sum
 

	
 

	
 
class DiscountAndQuantity(object):
 
    def __init__(self, discount, clause, quantity):
 
        self.discount = discount
 
        self.clause = clause
 
        self.quantity = quantity
 

	
 
    def __repr__(self):
 
        print "(discount=%s, clause=%s, quantity=%d)" % (
 
        return "(discount=%s, clause=%s, quantity=%d)" % (
 
            self.discount, self.clause, self.quantity,
 
        )
 

	
 

	
 
def available_discounts(user, categories, products):
 
    ''' Returns all discounts available to this user for the given categories
 
    and products. The discounts also list the available quantity for this user,
 
    not including products that are pending purchase. '''
 

	
 
    # discounts that match provided categories
 
    category_discounts = rego.DiscountForCategory.objects.filter(
 
        category__in=categories
 
    )
 
    # discounts that match provided products
 
    product_discounts = rego.DiscountForProduct.objects.filter(
 
        product__in=products
 
    )
 
    # discounts that match categories for provided products
 
    product_category_discounts = rego.DiscountForCategory.objects.filter(
 
        category__in=(product.category for product in products)
 
    )
 
    # (Not relevant: discounts that match products in provided categories)
 

	
 
    product_discounts = product_discounts.select_related(
 
        "product",
 
        "product__category",
 
    )
 

	
 
    all_category_discounts = category_discounts | product_category_discounts
 
    all_category_discounts = all_category_discounts.select_related(
 
        "category",
 
    )
 

	
 
    # The set of all potential discounts
 
    potential_discounts = set(itertools.chain(
 
        product_discounts,
 
        category_discounts,
 
        product_category_discounts,
 
        all_category_discounts,
 
    ))
 

	
 
    discounts = []
 

	
 
    # Markers so that we don't need to evaluate given conditions more than once
 
    accepted_discounts = set()
 
    failed_discounts = set()
 

	
 

	
 
    for discount in potential_discounts:
 
        real_discount = rego.DiscountBase.objects.get_subclass(
 
            pk=discount.discount.pk,
 
        )
 
        cond = ConditionController.for_condition(real_discount)
 

	
 
        # Count the past uses of the given discount item.
 
        # If this user has exceeded the limit for the clause, this clause
 
        # is not available any more.
 
        past_uses = rego.DiscountItem.objects.filter(
 
            cart__user=user,
 
            cart__active=False,  # Only past carts count
 
            cart__released=False,  # You can reuse refunded discounts
 
            discount=discount.discount,
 
            discount=real_discount,
 
        )
 
        agg = past_uses.aggregate(Sum("quantity"))
 
        past_use_count = agg["quantity__sum"]
 
        if past_use_count is None:
 
            past_use_count = 0
 

	
 
        if past_use_count >= discount.quantity:
 
            # This clause has exceeded its use count
 
            pass
 
        elif real_discount not in failed_discounts:
 
            # This clause is still available
 
            if real_discount in accepted_discounts or cond.is_met(user):
 
                # This clause is valid for this user
 
                discounts.append(DiscountAndQuantity(
 
                    discount=real_discount,
 
                    clause=discount,
 
                    quantity=discount.quantity - past_use_count,
 
                ))
 
                accepted_discounts.add(real_discount)
 
            else:
 
                # This clause is not valid for this user
 
                failed_discounts.add(real_discount)
 
    return discounts
registrasion/controllers/product.py
Show inline comments
...
 
@@ -2,60 +2,67 @@ import itertools
 

	
 
from django.db.models import Sum
 
from registrasion import models as rego
 

	
 
from category import CategoryController
 
from conditions import ConditionController
 

	
 

	
 
class ProductController(object):
 

	
 
    def __init__(self, product):
 
        self.product = product
 

	
 
    @classmethod
 
    def available_products(cls, user, category=None, products=None):
 
        ''' Returns a list of all of the products that are available per
 
        enabling conditions from the given categories.
 
        TODO: refactor so that all conditions are tested here and
 
        can_add_with_enabling_conditions calls this method. '''
 
        if category is None and products is None:
 
            raise ValueError("You must provide products or a category")
 

	
 
        if category is not None:
 
            all_products = rego.Product.objects.filter(category=category)
 
            all_products = all_products.select_related("category")
 
        else:
 
            all_products = []
 

	
 
        if products is not None:
 
            all_products = itertools.chain(all_products, products)
 
            all_products = set(itertools.chain(all_products, products))
 

	
 
        cat_quants = dict(
 
            (
 
                category,
 
                CategoryController(category).user_quantity_remaining(user),
 
            )
 
            for category in set(product.category for product in all_products)
 
        )
 

	
 
        passed_limits = set(
 
            product
 
            for product in all_products
 
            if CategoryController(product.category).user_quantity_remaining(
 
                user
 
            ) > 0
 
            if cat_quants[product.category] > 0
 
            if cls(product).user_quantity_remaining(user) > 0
 
        )
 

	
 
        failed_and_messages = ConditionController.test_enabling_conditions(
 
            user, products=passed_limits
 
        )
 
        failed_conditions = set(i[0] for i in failed_and_messages)
 

	
 
        out = list(passed_limits - failed_conditions)
 
        out.sort(key=lambda product: product.order)
 

	
 
        return out
 

	
 
    def user_quantity_remaining(self, user):
 
        ''' Returns the quantity of this product that the user add in the
 
        current cart. '''
 

	
 
        prod_limit = self.product.limit_per_user
 

	
 
        if prod_limit is None:
 
            # Don't need to run the remaining queries
 
            return 999999  # We can do better
 

	
 
        carts = rego.Cart.objects.filter(
registrasion/templatetags/registrasion_tags.py
Show inline comments
...
 
@@ -9,54 +9,55 @@ register = template.Library()
 

	
 
ProductAndQuantity = namedtuple("ProductAndQuantity", ["product", "quantity"])
 

	
 

	
 
@register.assignment_tag(takes_context=True)
 
def available_categories(context):
 
    ''' Returns all of the available product categories '''
 
    return CategoryController.available_categories(context.request.user)
 

	
 

	
 
@register.assignment_tag(takes_context=True)
 
def invoices(context):
 
    ''' Returns all of the invoices that this user has. '''
 
    return rego.Invoice.objects.filter(cart__user=context.request.user)
 

	
 

	
 
@register.assignment_tag(takes_context=True)
 
def items_pending(context):
 
    ''' Returns all of the items that this user has in their current cart,
 
    and is awaiting payment. '''
 

	
 
    all_items = rego.ProductItem.objects.filter(
 
        cart__user=context.request.user,
 
        cart__active=True,
 
    )
 
    ).select_related("product", "product__category")
 
    return all_items
 

	
 

	
 
@register.assignment_tag(takes_context=True)
 
def items_purchased(context, category=None):
 
    ''' Returns all of the items that this user has purchased, optionally
 
    from the given category. '''
 

	
 
    all_items = rego.ProductItem.objects.filter(
 
        cart__user=context.request.user,
 
        cart__active=False,
 
    )
 
        cart__released=False,
 
    ).select_related("product", "product__category")
 

	
 
    if category:
 
        all_items = all_items.filter(product__category=category)
 

	
 
    products = set(item.product for item in all_items)
 
    pq = all_items.values("product").annotate(quantity=Sum("quantity")).all()
 
    products = rego.Product.objects.all()
 
    out = []
 
    for product in products:
 
        pp = all_items.filter(product=product)
 
        quantity = pp.aggregate(Sum("quantity"))["quantity__sum"]
 
        out.append(ProductAndQuantity(product, quantity))
 
    for item in pq:
 
        prod = products.get(pk=item["product"])
 
        out.append(ProductAndQuantity(prod, item["quantity"]))
 
    return out
 

	
 

	
 
@register.filter
 
def multiply(value, arg):
 
    ''' Multiplies value by arg '''
 
    return value * arg
registrasion/views.py
Show inline comments
...
 
@@ -94,53 +94,62 @@ def guided_registration(request, page_id=0):
 
        # We're selling products
 

	
 
        last_category = attendee.highest_complete_category
 

	
 
        # Get the next category
 
        cats = rego.Category.objects
 
        cats = cats.filter(id__gt=last_category).order_by("order")
 

	
 
        if cats.count() == 0:
 
            # We've filled in every category
 
            attendee.completed_registration = True
 
            attendee.save()
 
            return next_step
 

	
 
        if last_category == 0:
 
            # Only display the first Category
 
            title = "Select ticket type"
 
            current_step = 2
 
            cats = [cats[0]]
 
        else:
 
            # Set title appropriately for remaining categories
 
            current_step = 3
 
            title = "Additional items"
 

	
 
        all_products = rego.Product.objects.filter(
 
            category__in=cats,
 
        ).select_related("category")
 

	
 
        available_products = set(ProductController.available_products(
 
            request.user,
 
            products=all_products,
 
        ))
 

	
 
        for category in cats:
 
            products = ProductController.available_products(
 
                request.user,
 
                category=category,
 
            )
 
            products = [
 
                i for i in available_products
 
                if i.category == category
 
            ]
 

	
 
            prefix = "category_" + str(category.id)
 
            p = handle_products(request, category, products, prefix)
 
            products_form, discounts, products_handled = p
 

	
 
            section = GuidedRegistrationSection(
 
                title=category.name,
 
                description=category.description,
 
                discounts=discounts,
 
                form=products_form,
 
            )
 
            if products:
 
                # This product category does not exist for this user
 
                sections.append(section)
 

	
 
            if request.method == "POST" and not products_form.errors:
 
                if category.id > attendee.highest_complete_category:
 
                    # This is only saved if we pass each form with no errors.
 
                    attendee.highest_complete_category = category.id
 

	
 
    if sections and request.method == "POST":
 
        for section in sections:
 
            if section.form.errors:
 
                break
...
 
@@ -259,93 +268,99 @@ def product_category(request, category_id):
 

	
 
    data = {
 
        "category": category,
 
        "discounts": discounts,
 
        "form": products_form,
 
        "voucher_form": voucher_form,
 
    }
 

	
 
    return render(request, "registrasion/product_category.html", data)
 

	
 

	
 
def handle_products(request, category, products, prefix):
 
    ''' Handles a products list form in the given request. Returns the
 
    form instance, the discounts applicable to this form, and whether the
 
    contents were handled. '''
 

	
 
    current_cart = CartController.for_user(request.user)
 

	
 
    ProductsForm = forms.ProductsForm(category, products)
 

	
 
    # Create initial data for each of products in category
 
    items = rego.ProductItem.objects.filter(
 
        product__in=products,
 
        cart=current_cart.cart,
 
    )
 
    ).select_related("product")
 
    quantities = []
 
    for product in products:
 
        # Only add items that are enabled.
 
        try:
 
            quantity = items.get(product=product).quantity
 
        except ObjectDoesNotExist:
 
            quantity = 0
 
        quantities.append((product, quantity))
 
    seen = set()
 

	
 
    for item in items:
 
        quantities.append((item.product, item.quantity))
 
        seen.add(item.product)
 

	
 
    zeros = set(products) - seen
 
    for product in zeros:
 
        quantities.append((product, 0))
 

	
 
    products_form = ProductsForm(
 
        request.POST or None,
 
        product_quantities=quantities,
 
        prefix=prefix,
 
    )
 

	
 
    if request.method == "POST" and products_form.is_valid():
 
        if products_form.has_changed():
 
            set_quantities_from_products_form(products_form, current_cart)
 

	
 
        # If category is required, the user must have at least one
 
        # in an active+valid cart
 
        if category.required:
 
            carts = rego.Cart.objects.filter(user=request.user)
 
            items = rego.ProductItem.objects.filter(
 
                product__category=category,
 
                cart=carts,
 
            )
 
            if len(items) == 0:
 
                products_form.add_error(
 
                    None,
 
                    "You must have at least one item from this category",
 
                )
 
    handled = False if products_form.errors else True
 

	
 
    discounts = discount.available_discounts(request.user, [], products)
 

	
 
    return products_form, discounts, handled
 

	
 

	
 
def set_quantities_from_products_form(products_form, current_cart):
 

	
 
    quantities = list(products_form.product_quantities())
 

	
 
    pks = [i[0] for i in quantities]
 
    products = rego.Product.objects.filter(id__in=pks).select_related("category")
 

	
 
    product_quantities = [
 
        (rego.Product.objects.get(pk=i[0]), i[1]) for i in quantities
 
        (products.get(pk=i[0]), i[1]) for i in quantities
 
    ]
 
    field_names = dict(
 
        (i[0][0], i[1][2]) for i in zip(product_quantities, quantities)
 
    )
 

	
 
    try:
 
        current_cart.set_quantities(product_quantities)
 
    except CartValidationError as ve:
 
        for ve_field in ve.error_list:
 
            product, message = ve_field.message
 
            if product in field_names:
 
                field = field_names[product]
 
            elif isinstance(product, rego.Product):
 
                continue
 
            else:
 
                field = None
 
            products_form.add_error(field, message)
 

	
 

	
 
def handle_voucher(request, prefix):
 
    ''' Handles a voucher form in the given request. Returns the voucher
 
    form instance, and whether the voucher code was handled. '''
 

	
 
    voucher_form = forms.VoucherForm(request.POST or None, prefix=prefix)
0 comments (0 inline, 0 general)