From 74df61400793f2a66355541de8f2103a1f457346 Mon Sep 17 00:00:00 2001 From: Baptiste Jonglez Date: Sun, 1 Jan 2017 21:17:47 +0100 Subject: [PATCH 1/2] Optimise SQL queries when displaying bills for a project By defaut, SQLAlchemy uses lazy loading, which means that displaying n bills will generate around n queries (to get the list of owers of each bill). Pre-load the list of owers to drastically decrease the number of SQL queries. Before this commit: 1004 SQL queries, 7535 ms elapsed time, 7536 ms CPU time, 530 ms SQL time After this commit: 5 SQL queries, 3342 ms elapsed time, 3393 ms CPU time, 15 ms SQL time Measured request: display the list of all bills for the project (without displaying the sidebar with balances) Test setup to measure performance improvement: - 5 users with various weights - 1000 bills, each paid by a random user, each involving all 5 users - laptop with Celeron N2830@2.16 GHz, SSD Samsung 850 EVO - sqlite database on SSD, using sqlite 3.15.2 - python 2.7.13 - Flask-DebugToolbar 0.10.0 (to count SQL queries and loading time) Performance measurements (using Flask-DebugToolbar with the second request, to avoid measuring cold-cache performance): - number of SQL queries - elapsed time (from request to response) - total CPU time consumed by the server handling the request - total time spent on SQL queries (as reported by SQLAlchemy) --- budget/web.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/budget/web.py b/budget/web.py index 63fbe4d..87aef26 100644 --- a/budget/web.py +++ b/budget/web.py @@ -15,6 +15,7 @@ from flask.ext.mail import Mail, Message from flask.ext.babel import get_locale, gettext as _ from smtplib import SMTPRecipientsRefused import werkzeug +from sqlalchemy import orm # local modules from models import db, Project, Person, Bill @@ -277,7 +278,8 @@ def list_bills(): # set the last selected payer as default choice if exists if 'last_selected_payer' in session: bill_form.payer.data = session['last_selected_payer'] - bills = g.project.get_bills() + # Preload the "owers" relationship for all bills + bills = g.project.get_bills().options(orm.subqueryload(Bill.owers)) return render_template("list_bills.html", bills=bills, member_form=MemberForm(g.project), From 8e5ac8ae98824b8b359cc0f020b10acb29985dab Mon Sep 17 00:00:00 2001 From: Baptiste Jonglez Date: Sun, 1 Jan 2017 22:32:32 +0100 Subject: [PATCH 2/2] Optimise SQL queries when computing balance This avoids creating thousands of small SQL queries when computing the balance of users. This significantly improves the performance of displaying the main page of a project, since the balance of users is displayed there: Before this commit: 4004 SQL queries, 19793 ms elapsed time, 19753 ms CPU time, 2094 ms SQL time After this commit: 12 SQL queries, 3688 ms elapsed time, 3753 ms CPU time, 50 ms SQL time Measured request: display the sidebar with the balance of all users for the project (without displaying the list of bills) This commit also greatly improves the performance of the "settle bills" page: Before this commit: 8006 SQL queries, 39167 ms elapsed time, 39600 ms CPU time, 4141 ms SQL time After this commit: 22 SQL queries, 7144 ms elapsed time, 7283 ms CPU time, 96 ms SQL time Measured request: display the "Settle bills" page Test setup to measure performance improvement: - 5 users with various weights - 1000 bills, each paid by a random user, each involving all 5 users - laptop with Celeron N2830@2.16 GHz, SSD Samsung 850 EVO - sqlite database on SSD, using sqlite 3.15.2 - python 2.7.13 - Flask-DebugToolbar 0.10.0 (to count SQL queries and loading time) Performance measurements (using Flask-DebugToolbar on the second request, to avoid measuring cold-cache performance): - number of SQL queries - elapsed time (from request to response) - total CPU time consumed by the server handling the request - total time spent on SQL queries (as reported by SQLAlchemy) --- budget/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/budget/models.py b/budget/models.py index 852b3e1..88d44ae 100644 --- a/budget/models.py +++ b/budget/models.py @@ -37,7 +37,7 @@ class Project(db.Model): # for each person for person in self.members: # get the list of bills he has to pay - bills = Bill.query.filter(Bill.owers.contains(person)) + bills = Bill.query.options(orm.subqueryload(Bill.owers)).filter(Bill.owers.contains(person)) for bill in bills.all(): if person != bill.payer: share = bill.pay_each() * person.weight