Merge pull request #154 from willstott101/perf1

Various performance improvements.

core/models.py

@@ -35,8 +35,7 @@ class ImageManager(models.Manager):
         # a chance of getting Database into a inconsistent state when we
         # try to create thumbnails one by one later
         image = self.create(image=obj)
-        for size in settings.IMAGE_SIZES.keys():
-            Thumbnail.objects.get_or_create_at_size(image.pk, size)
+        Thumbnail.objects.get_or_create_at_sizes(image, settings.IMAGE_SIZES.keys())
         return image
 
 

core/serializers.py

@@ -52,8 +52,7 @@ class ImageSerializer(serializers.ModelSerializer):
 
     def create(self, validated_data):
         image = super(ImageSerializer, self).create(validated_data)
-        for size in settings.IMAGE_SIZES:
-            Thumbnail.objects.get_or_create_at_size(image.pk, size)
+        Thumbnail.objects.get_or_create_at_sizes(image, settings.IMAGE_SIZES.keys())
         return image
 
 

core/tests/api.py

@@ -12,12 +12,6 @@ from .helpers import create_image, create_user, create_pin
 from core.models import Pin, Image
 
 
-def filter_generator_for(size):
-    def wrapped_func(obj):
-        return Thumbnail.objects.get_or_create_at_size(obj.pk, size)
-    return wrapped_func
-
-
 def mock_requests_get(url, **kwargs):
     response = mock.Mock(content=open('logo.png', 'rb').read())
     return response

core/tests/helpers.py

@@ -32,8 +32,7 @@ def create_tag(name):
 
 def create_image():
     image = Image.objects.create(image=ImageFile(open(TEST_IMAGE_PATH, 'rb')))
-    for size in settings.IMAGE_SIZES.keys():
-        Thumbnail.objects.get_or_create_at_size(image.pk, size)
+    Thumbnail.objects.get_or_create_at_sizes(image, settings.IMAGE_SIZES.keys())
     return image
 
 

core/views.py

@@ -23,7 +23,7 @@ class ImageViewSet(mixins.CreateModelMixin, GenericViewSet):
 
 
 class PinViewSet(viewsets.ModelViewSet):
-    queryset = Pin.objects.all()
+    queryset = Pin.objects.all().select_related('image', 'submitter')
     serializer_class = api.PinSerializer
     filter_backends = (DjangoFilterBackend, SearchFilter, OrderingFilter)
     filter_fields = ("submitter__username", 'tags__name', )

django_images/models.py

@@ -1,12 +1,10 @@
 import hashlib
 import os.path
-from io import BytesIO
 
 from django.db import models
 from django.core.files.uploadedfile import InMemoryUploadedFile
 from django.core.urlresolvers import reverse
 from django.dispatch import receiver
-import PIL
 
 try:
     from importlib import import_module
@@ -67,38 +65,37 @@ class Image(models.Model):
 
 
 class ThumbnailManager(models.Manager):
-    def get_or_create_at_size(self, image_id, size):
-        image = Image.objects.get(id=image_id)
-        if size not in IMAGE_SIZES:
-            raise ValueError("Received unknown size: %s" % size)
-        try:
-            thumbnail = image.get_by_size(size)
-        except Thumbnail.DoesNotExist:
-            img = utils.scale_and_crop(image.image, **IMAGE_SIZES[size])
-            # save to memory
-            buf = BytesIO()
+    def get_or_create_at_sizes(self, image, sizes):
+        sizes_to_create = list(sizes)
+        sized = {}
+        for size in sizes:
+            if size not in IMAGE_SIZES:
+                raise ValueError("Received unknown size: %s" % size)
+
             try:
-                img.save(buf, img.format, **img.info)
-            except IOError:
-                if img.info.get('progression'):
-                    orig_MAXBLOCK = PIL.ImageFile.MAXBLOCK
-                    temp_MAXBLOCK = 1048576
-                    if orig_MAXBLOCK >= temp_MAXBLOCK:
-                        raise
-                    PIL.ImageFile.MAXBLOCK = temp_MAXBLOCK
-                    try:
-                        img.save(buf, img.format, **img.info)
-                    finally:
-                        PIL.ImageFile.MAXBLOCK = orig_MAXBLOCK
-                else:
-                    raise
-            # and save to storage
-            original_dir, original_file = os.path.split(image.image.name)
-            thumb_file = InMemoryUploadedFile(buf, "image", original_file,
-                                              None, buf.tell(), None)
-            thumbnail, created = image.thumbnail_set.get_or_create(
-                size=size, defaults={'image': thumb_file})
-        return thumbnail
+                sized[size] = image.get_by_size(size)
+            except Thumbnail.DoesNotExist:
+                pass
+            else:
+                sizes_to_create.remove(size)
+
+        if sizes_to_create:
+            bufs = [
+                utils.write_image_in_memory(img)
+                for img in utils.scale_and_crop_iter(
+                    image.image,
+                    [IMAGE_SIZES[size] for size in sizes_to_create])
+            ]
+            for size, buf in zip(sizes_to_create, bufs):
+                # and save to storage
+                original_dir, original_file = os.path.split(image.image.name)
+                thumb_file = InMemoryUploadedFile(buf, "image", original_file,
+                                                  None, buf.tell(), None)
+                sized[size], created = image.thumbnail_set.get_or_create(
+                    size=size, defaults={'image': thumb_file})
+
+        # Make sure this is in the correct order
+        return [sized[size] for size in sizes]
 
 
 class Thumbnail(models.Model):

django_images/tests.py

@@ -4,10 +4,10 @@ from django.test import TestCase
 from django.core.files.images import ImageFile
 from django.conf import settings
 from django.utils.six import BytesIO
-from django.core.urlresolvers import reverse
 from django_images.models import Image, Thumbnail
 from django_images.templatetags.images import at_size
-from django_images.utils import scale_and_crop
+from django_images.utils import scale_and_crop_single
+from PIL import Image as PILImage
 
 
 class ImageModelTest(TestCase):
@@ -20,7 +20,7 @@ class ImageModelTest(TestCase):
 
     def test_get_by_size(self):
         size = list(settings.IMAGE_SIZES.keys())[0]
-        Thumbnail.objects.get_or_create_at_size(self.image.id, size)
+        Thumbnail.objects.get_or_create_at_sizes(self.image, [size])
         self.image.get_by_size(size)
 
     def test_get_absolute_url(self):
@@ -28,14 +28,9 @@ class ImageModelTest(TestCase):
         self.assertEqual(url, self.image.image.url)
         # For thumbnail
         size = list(settings.IMAGE_SIZES.keys())[0]
-        thumb = Thumbnail.objects.get_or_create_at_size(self.image.id, size)
+        thumb = Thumbnail.objects.get_or_create_at_sizes(self.image, [size])[0]
         url = self.image.get_absolute_url(size)
         self.assertEqual(url, thumb.image.url)
-        # Fallback on creation url
-        size = list(settings.IMAGE_SIZES.keys())[1]
-        url = self.image.get_absolute_url(size)
-        fallback_url = reverse('image-thumbnail', args=(self.image.id, size))
-        self.assertEqual(url, fallback_url)
 
 
 class ThumbnailManagerModelTest(TestCase):
@@ -48,17 +43,17 @@ class ThumbnailManagerModelTest(TestCase):
         self.size = list(settings.IMAGE_SIZES.keys())[0]
 
     def test_unknown_size(self):
-        self.assertRaises(ValueError, Thumbnail.objects.get_or_create_at_size,
-                          self.image.id, 'foo')
+        self.assertRaises(ValueError, Thumbnail.objects.get_or_create_at_sizes,
+                          self.image, ['foo'])
 
     # TODO: Test the image object and data
     def test_create(self):
-        Thumbnail.objects.get_or_create_at_size(self.image.id, self.size)
+        Thumbnail.objects.get_or_create_at_sizes(self.image, [self.size])
         self.assertEqual(self.image.thumbnail_set.count(), 1)
 
     def test_get(self):
-        thumb = Thumbnail.objects.get_or_create_at_size(self.image.id, self.size)
-        thumb2 = Thumbnail.objects.get_or_create_at_size(self.image.id, self.size)
+        thumb = Thumbnail.objects.get_or_create_at_sizes(self.image, [self.size])[0]
+        thumb2 = Thumbnail.objects.get_or_create_at_sizes(self.image, [self.size])[0]
         self.assertEqual(thumb.id, thumb2.id)
 
 
@@ -70,7 +65,7 @@ class ThumbnailModelTest(TestCase):
         self.image = Image.objects.create(width=370, height=370,
                                           image=ImageFile(image_obj, '01.png'))
         size = list(settings.IMAGE_SIZES.keys())[0]
-        self.thumb = Thumbnail.objects.get_or_create_at_size(self.image.id, size)
+        self.thumb = Thumbnail.objects.get_or_create_at_sizes(self.image, [size])[0]
 
     def test_get_absolute_url(self):
         url = self.thumb.get_absolute_url()
@@ -85,11 +80,11 @@ class PostSaveSignalOriginalChangedTestCase(TestCase):
         self.image = Image.objects.create(width=370, height=370,
                                           image=ImageFile(image_obj, '01.png'))
         size = list(settings.IMAGE_SIZES.keys())[0]
-        self.thumb = Thumbnail.objects.get_or_create_at_size(self.image.id, size)
+        self.thumb = Thumbnail.objects.get_or_create_at_sizes(self.image, [size])[0]
 
     def test_post_save_signal_original_changed(self):
         size = list(settings.IMAGE_SIZES.keys())[0]
-        Thumbnail.objects.get_or_create_at_size(self.image.id, size)
+        Thumbnail.objects.get_or_create_at_sizes(self.image, [size])
         self.image.delete()
         self.assertFalse(Thumbnail.objects.exists())
 
@@ -102,7 +97,7 @@ class PostDeleteSignalDeleteImageFileTest(TestCase):
         self.image = Image.objects.create(width=370, height=370,
                                           image=ImageFile(image_obj, '01.png'))
         size = list(settings.IMAGE_SIZES.keys())[0]
-        self.thumb = Thumbnail.objects.get_or_create_at_size(self.image.id, size)
+        self.thumb = Thumbnail.objects.get_or_create_at_sizes(self.image, [size])[0]
 
     @mock.patch('django_images.models.IMAGE_AUTO_DELETE', True)
     def test_post_delete_signal_delete_image_files_enabled(self):
@@ -136,7 +131,7 @@ class AtSizeTemplateTagTest(TestCase):
         self.image = Image.objects.create(width=370, height=370,
                                           image=ImageFile(image_obj, '01.png'))
         size = list(settings.IMAGE_SIZES.keys())[0]
-        self.thumb = Thumbnail.objects.get_or_create_at_size(self.image.id, size)
+        self.thumb = Thumbnail.objects.get_or_create_at_sizes(self.image, [size])[0]
 
     def test_at_size(self):
         size = list(settings.IMAGE_SIZES.keys())[0]
@@ -144,65 +139,37 @@ class AtSizeTemplateTagTest(TestCase):
         self.assertEqual(url, self.thumb.image.url)
 
 
-class ThumbnailViewTest(TestCase):
-    def setUp(self):
-        image_obj = BytesIO()
-        qrcode_obj = qrcode.make('https://mirumee.com/')
-        qrcode_obj.save(image_obj)
-        self.image = Image.objects.create(width=370, height=370,
-                                          image=ImageFile(image_obj, '01.png'))
-        self.size = list(settings.IMAGE_SIZES.keys())[0]
-        self.thumb = Thumbnail.objects.get_or_create_at_size(self.image.id, self.size)
-
-    def test_redirect(self):
-        url = reverse('image-thumbnail', args=[self.image.id, self.size])
-        response = self.client.get(url)
-        self.assertNotEqual(url, self.thumb.image.url)
-        self.assertEqual(response.status_code, 302)
-        self.assertEqual(response.url, self.thumb.image.url)
-
-    def test_not_found(self):
-        url = reverse('image-thumbnail', args=['42', self.size])
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 404)
-
-    def test_size_not_found(self):
-        url = reverse('image-thumbnail', args=[self.image.id, '42'])
-        response = self.client.get(url)
-        self.assertEqual(response.status_code, 404)
-
-
 class UtilsScaleAndDropTest(TestCase):
     def setUp(self):
         image_obj = BytesIO()
         qrcode_obj = qrcode.make('https://mirumee.com/')
         qrcode_obj.save(image_obj)
-        self.imagefile = ImageFile(image_obj, '01.png')
+        self.image = PILImage.open(image_obj)
 
     def test_change_size(self):
         new_size = (10, 10)
-        image = scale_and_crop(self.imagefile, new_size)
-        self.assertEqual(new_size, image.im.size)
+        image = scale_and_crop_single(self.image, new_size)
+        self.assertEqual(new_size, image.size)
 
     def test_crop(self):
         new_size = (10, 10)
-        image = scale_and_crop(self.imagefile, new_size, crop=True)
-        self.assertEqual(new_size, image.im.size)
+        image = scale_and_crop_single(self.image, new_size, crop=True)
+        self.assertEqual(new_size, image.size)
 
     def test_disabled_upscale(self):
-        image = scale_and_crop(self.imagefile, (740, 740), upscale=False)
-        self.assertLess(image.im.size[0], 371)
-        self.assertLess(image.im.size[1], 371)
+        image = scale_and_crop_single(self.image, (740, 740), upscale=False)
+        self.assertLess(image.size[0], 371)
+        self.assertLess(image.size[1], 371)
 
     def test_enaabled_upscale(self):
-        image = scale_and_crop(self.imagefile, (740, 740), upscale=True)
-        self.assertGreater(image.im.size[0], 371)
-        self.assertGreater(image.im.size[1], 371)
+        image = scale_and_crop_single(self.image, (740, 740), upscale=True)
+        self.assertGreater(image.size[0], 371)
+        self.assertGreater(image.size[1], 371)
 
     def test_not_change_quality(self):
-        image = scale_and_crop(self.imagefile, (10, 10), quality=None)
+        image = scale_and_crop_single(self.image, (10, 10), quality=None)
         self.assertEqual(image.info.get('quality'), None)
 
     def test_change_quality(self):
-        image = scale_and_crop(self.imagefile, (10, 10), quality=50)
+        image = scale_and_crop_single(self.image, (10, 10), quality=50)
         self.assertEqual(image.info.get('quality'), 50)

django_images/urls.py

@@ -1,6 +0,0 @@
-from django.conf.urls import url
-from . import views
-
-urlpatterns = [
-    url(r'^thumbnail/(?P<image_id>\d+)/(?P<size>[^/]+)/$', views.thumbnail, name='image-thumbnail'),
-]

django_images/utils.py

@@ -1,13 +1,45 @@
+from contextlib import contextmanager
+from io import BytesIO
+import PIL
 from PIL import Image
 
 
+@contextmanager
+def open_django_file(fieldfile):
+    fieldfile.open()
+    try:
+        yield fieldfile
+    finally:
+        fieldfile.close()
+
+
+def scale_and_crop_iter(image, options):
+    """
+    Generator which will yield several variations on the input image.
+    Resize, crop and/or change quality of image.
+
+    :param image: Source image file
+    :param type: :class:`django.core.files.images.ImageFile
+
+    :param`options: List of option dictionaries, See scale_and_crop_single
+    argument names for available keys.
+    :type options: list of dict
+    """
+    with open_django_file(image) as img:
+        im = Image.open(img)
+        im.load()
+        for opts in options:
+            # Use already-loaded file when cropping.
+            yield scale_and_crop_single(im, **opts)
+
+
 # this neat function is based on easy-thumbnails
-def scale_and_crop(image, size, crop=False, upscale=False, quality=None):
+def scale_and_crop_single(image, size, crop=False, upscale=False, quality=None):
     """
     Resize, crop and/or change quality of an image.
 
     :param image: Source image file
-    :param type: :class:`django.core.files.images.ImageFile`
+    :param type: :class:`PIL.Image`
 
     :param size: Size as width & height, zero as either means unrestricted
     :type size: tuple of two int
@@ -24,15 +56,7 @@ def scale_and_crop(image, size, crop=False, upscale=False, quality=None):
     :return: Handled image
     :rtype: class:`PIL.Image`
     """
-    # Open image and store format/metadata.
-    image.open()
-    im = Image.open(image)
-    im_format, im_info = im.format, im.info
-    if quality:
-        im_info['quality'] = quality
-
-    # Force PIL to load image data.
-    im.load()
+    im = image
 
     source_x, source_y = [float(v) for v in im.size]
     target_x, target_y = [float(v) for v in size]
@@ -68,6 +92,31 @@ def scale_and_crop(image, size, crop=False, upscale=False, quality=None):
             im = im.crop(box)
 
     # Close image and replace format/metadata, as PIL blows this away.
-    im.format, im.info = im_format, im_info
-    image.close()
+    # We mutate the quality, but needs to passed into save() to actually
+    # do anything.
+    info = image.info
+    if quality is not None:
+        info['quality'] = quality
+    im.format, im.info = image.format, info
     return im
+
+
+def write_image_in_memory(img):
+    # save to memory
+    buf = BytesIO()
+    try:
+        img.save(buf, img.format, **img.info)
+    except IOError:
+        if img.info.get('progression'):
+            orig_MAXBLOCK = PIL.ImageFile.MAXBLOCK
+            temp_MAXBLOCK = 1048576
+            if orig_MAXBLOCK >= temp_MAXBLOCK:
+                raise
+            PIL.ImageFile.MAXBLOCK = temp_MAXBLOCK
+            try:
+                img.save(buf, img.format, **img.info)
+            finally:
+                PIL.ImageFile.MAXBLOCK = orig_MAXBLOCK
+        else:
+            raise
+    return buf

django_images/views.py

@@ -1,14 +0,0 @@
-from django.http import HttpResponseNotFound
-from django.shortcuts import get_object_or_404, redirect
-
-from . import models
-from .settings import IMAGE_SIZES
-
-
-def thumbnail(request, image_id, size):
-    image = get_object_or_404(models.Image, id=image_id)
-    if size not in IMAGE_SIZES:
-        return HttpResponseNotFound()
-
-    return redirect(models.Thumbnail.objects.get_or_create_at_size(image.id,
-                                                                   size))

pinry/urls.py

@@ -32,7 +32,3 @@ if settings.DEBUG:
 
 if settings.IS_TEST:
     urlpatterns += staticfiles_urlpatterns()
-    # For test running of django_images
-    urlpatterns += [
-        url(r'^__images/', include('django_images.urls')),
-    ]