Optimize some db queries

Basically make django generate the joins/prefetches/subqueries instead
of firing of hundreds of queries on the db. Probably doesn't actually
matter that much given how seldom these queries are run, but still.

diff --git a/hamnadmin/hamnadmin/register/management/commands/aggregate_feeds.py b/hamnadmin/hamnadmin/register/management/commands/aggregate_feeds.py
index f617fa87bed53d358f5f4a0594cd83c9b44e5399..a4f6a85a36621dcb55051a227edde91f6d220290 100644 (file)
--- a/hamnadmin/hamnadmin/register/management/commands/aggregate_feeds.py
+++ b/hamnadmin/hamnadmin/register/management/commands/aggregate_feeds.py
@@ -3,7 +3,7 @@ import gevent
 
 from django.core.management.base import BaseCommand
 from django.db import transaction
-from django.db.models import Q
+from django.db.models import Q, Exists, OuterRef
 from django.conf import settings
 
 from datetime import datetime
@@ -44,6 +44,9 @@ class Command(BaseCommand):
             # Fetch all feeds - that are not archived. We do fetch feeds that are not approved,
             # to make sure they work.
             feeds = Blog.objects.filter(archived=False)
+        feeds = feeds.annotate(
+            has_entries=Exists(Post.objects.filter(feed=OuterRef("pk"), hidden=False)),
+        )
 
         # Fan out the fetching itself
         fetchers = [FeedFetcher(f, self.trace) for f in feeds]

diff --git a/hamnadmin/hamnadmin/register/models.py b/hamnadmin/hamnadmin/register/models.py
index 6377a8a06a98963f2cecd3f5fb53db18eb111268..1ce886daf9b60c9c89e4588a2e437e078e64ca69 100644 (file)
--- a/hamnadmin/hamnadmin/register/models.py
+++ b/hamnadmin/hamnadmin/register/models.py
@@ -49,20 +49,6 @@ class Blog(models.Model):
     def email(self):
         return self.user.email
 
-    @property
-    def recent_failures(self):
-        return self.aggregatorlog_set.filter(success=False, ts__gt=datetime.now() - timedelta(days=1)).count()
-
-    def last_was_error(self):
-        if self.lastsuccess:
-            return self.aggregatorlog_set.filter(success=False, ts__gt=self.lastsuccess).exists()
-        else:
-            return self.aggregatorlog_set.filter(success=False).exists()
-
-    @property
-    def has_entries(self):
-        return self.posts.filter(hidden=False).exists()
-
     @property
     def latestentry(self):
         try:

diff --git a/hamnadmin/hamnadmin/register/templates/index.html b/hamnadmin/hamnadmin/register/templates/index.html
index 424e620a2f8503cee9588b14032650364e2d2e18..526b8d6fcd05802612649011bf6cc0f636ad3222 100644 (file)
--- a/hamnadmin/hamnadmin/register/templates/index.html
+++ b/hamnadmin/hamnadmin/register/templates/index.html
@@ -53,8 +53,8 @@ Last successful http fetch dated: {{blog.lastget|date:"Y-m-d H:i:s"}}<br/>
     {%endif%}
       </div>
       <div>
-      {% if blog.last_was_error%}<span class="label label-danger">Last attempt failed</span>
-      {%else%}<span class="label label-success">Last attempt succeeded</span>{%endif%}
+      {% if blog.last_was_success%}<span class="label label-success">Last attempt succeeded</span>
+      {%else%}<span class="label label-danger">Last attempt failed</span>{%endif%}
    {%else%}
     <span class="label label-success">Approved and working</span>
    {%endif%}{#recent_failures#}

diff --git a/hamnadmin/hamnadmin/register/views.py b/hamnadmin/hamnadmin/register/views.py
index 0a69713b20e1ffec33550c684758a43b9c0008fe..832f4ae91ec48ce87cbe12cf76843a47b60c2882 100644 (file)
--- a/hamnadmin/hamnadmin/register/views.py
+++ b/hamnadmin/hamnadmin/register/views.py
@@ -3,7 +3,7 @@ from django.shortcuts import render, get_object_or_404
 from django.contrib.auth.decorators import login_required, user_passes_test
 from django.conf import settings
 from django.db import transaction
-from django.db.models import Count, Max
+from django.db.models import Count, Max, Q, Subquery, OuterRef, Exists
 from django.contrib import messages
 
 from hamnadmin.register.models import Post, Blog, Team, AggregatorLog, AuditEntry
@@ -18,7 +18,7 @@ from .forms import BlogEditForm, ModerateRejectForm
 # Public planet
 def planet_home(request):
     statdate = datetime.datetime.now() - datetime.timedelta(days=61)
-    posts = Post.objects.filter(hidden=False, feed__approved=True).order_by('-dat')[:30]
+    posts = Post.objects.select_related('feed', 'feed__team').filter(hidden=False, feed__approved=True).order_by('-dat')[:30]
     topposters = Blog.objects.filter(approved=True, excludestats=False, posts__hidden=False, posts__dat__gt=statdate).annotate(numposts=Count('posts__id')).order_by('-numposts')[:10]
     topteams = Team.objects.filter(blog__approved=True, blog__excludestats=False, blog__posts__hidden=False, blog__posts__dat__gt=statdate).annotate(numposts=Count('blog__posts__id')).order_by('-numposts')[:10]
     return render(request, 'index.tmpl', {
@@ -49,12 +49,19 @@ def issuperuser(user):
 def root(request):
     isadmin = request.user.is_superuser and 'admin' in request.GET and request.GET['admin'] == '1'
     if isadmin:
-        blogs = Blog.objects.all().order_by('archived', 'approved', 'name')
+        blogs = Blog.objects.select_related('user').all()
     else:
-        blogs = Blog.objects.filter(user=request.user).order_by('archived', 'approved', 'name')
+        blogs = Blog.objects.filter(user=request.user)
+
+    blogs = blogs.annotate(
+        has_entries=Exists(Post.objects.filter(feed=OuterRef("pk"), hidden=False)),
+        recent_failures=Count('aggregatorlog', filter=Q(aggregatorlog__success=False, aggregatorlog__ts__gt=datetime.datetime.now() - datetime.timedelta(days=1))),
+        last_was_success=Subquery(AggregatorLog.objects.filter(feed=OuterRef("pk")).values('success')[:1]),
+    ).order_by('archived', 'approved', 'name')
+
     return render(request, 'index.html', {
         'blogs': blogs,
-        'teams': Team.objects.filter(manager=request.user).order_by('name'),
+        'teams': Team.objects.filter(manager=request.user).prefetch_related('blog_set').order_by('name'),
         'title': 'All blogs' if isadmin else 'Your blogs',
         'isadmin': isadmin,
     })
@@ -64,10 +71,14 @@ def root(request):
 @transaction.atomic
 def edit(request, id=None):
     if id:
+        blogqs = Blog.objects.all().annotate(
+            has_entries=Exists(Post.objects.filter(feed=OuterRef("pk"), hidden=False)),
+            recent_failures=Count('aggregatorlog', filter=Q(aggregatorlog__success=False, aggregatorlog__ts__gt=datetime.datetime.now() - datetime.timedelta(days=1))),
+        )
         if request.user.is_superuser:
-            blog = get_object_or_404(Blog, id=id)
+            blog = get_object_or_404(blogqs, id=id)
         else:
-            blog = get_object_or_404(Blog, id=id, user=request.user)
+            blog = get_object_or_404(blogqs, id=id, user=request.user)
     else:
         blog = Blog(user=request.user, name="{0} {1}".format(request.user.first_name, request.user.last_name))
 
@@ -269,7 +280,7 @@ def blogpost_delete(request, blogid, postid):
 @user_passes_test(issuperuser)
 def moderate(request):
     return render(request, 'moderate.html', {
-        'blogs': Blog.objects.filter(approved=False).annotate(oldest=Max('posts__dat')).order_by('oldest'),
+        'blogs': Blog.objects.filter(approved=False).select_related('user', 'team').annotate(oldest=Max('posts__dat')).order_by('oldest'),
         'title': 'Moderation',
     })