Add media URLs to Page Tree Api responses for offline access (#1612)

Co-authored-by: Sher Ali <[email protected]>
Co-authored-by: Ali Hassan <[email protected]>

Author: 3 people

home/mixins.py

@@ -1,6 +1,4 @@
-from bs4 import BeautifulSoup
 from django.utils.functional import cached_property
-from wagtail.images.models import Rendition
 
 
 class PageUtilsMixin:
@@ -32,39 +30,6 @@ def is_first_content(self):
     def get_type(self):
         return self.__class__.__name__.lower()
 
-    def _get_renditions(self, image_id):
-        image_urls = []
-        for rendition in Rendition.objects.filter(image_id=image_id):
-            image_urls.append(rendition.url)
-        return image_urls
-
-    def _get_images_from_block(self, value):
-        image_urls = []
-
-        tags = BeautifulSoup(value, "html.parser").find_all('embed')
-        for tag in tags:
-            if tag.attrs['embedtype'] == 'image':
-                image_urls += self._get_renditions(tag.attrs['id'])
-
-        return image_urls
-
-    def _get_stream_data_image_urls(self, raw_data):
-        image_urls = []
-
-        for block in raw_data:
-            if block['type'] == 'image':
-                image_urls += self._get_renditions(block['value'])
-            if block['type'] == 'paragraph':
-                image_urls += self._get_images_from_block(block['value'])
-            if block['type'] == 'download':
-                image_urls += self._get_images_from_block(block['value'].get('description', ''))
-
-        return image_urls
-
-    @property
-    def get_image_urls(self):
-        raise NotImplementedError
-
 
 class TitleIconMixin:
     """

home/models.py

@@ -30,28 +30,32 @@
 from wagtail.contrib.settings.models import BaseSetting
 from wagtail.contrib.settings.registry import register_setting
 from wagtail.core import blocks
-from wagtail.core.fields import StreamField, RichTextField
+from wagtail.core.fields import StreamField
 from wagtail.core.models import Orderable, Page, Site, Locale
 from wagtail.core.rich_text import get_text_for_indexing
 from wagtail.images.blocks import ImageChooserBlock
 from wagtail.images.edit_handlers import ImageChooserPanel
 from wagtail.images.models import Image
 from wagtail.search import index
 from wagtailmarkdown.blocks import MarkdownBlock
-from wagtailmenus.models import AbstractFlatMenuItem, BooleanField
+from wagtailmenus.models import AbstractFlatMenuItem
 from wagtailsvg.models import Svg
 from wagtailsvg.edit_handlers import SvgChooserPanel
 
 from messaging.blocks import ChatBotButtonBlock
-from comments.models import CommentableMixin, CannedResponse
-from .blocks import (
+from comments.models import CommentableMixin
+from home.blocks import (
     MediaBlock, SocialMediaLinkBlock, SocialMediaShareButtonBlock, EmbeddedPollBlock, EmbeddedSurveyBlock,
     EmbeddedQuizBlock, PageButtonBlock, NumberedListBlock, RawHTMLBlock, ArticleBlock, DownloadButtonBlock,
 )
 from .forms import SectionPageForm
 from .mixins import PageUtilsMixin, TitleIconMixin
 from .utils.image import convert_svg_to_png_bytes
 from .utils.progress_manager import ProgressManager
+from home.utils import (
+    collect_urls_from_streamfield,
+    get_all_renditions_urls,
+)
 import iogt.iogt_globals as globals_
 
 User = get_user_model()
@@ -91,8 +95,8 @@ def get_context(self, request):
         return context
 
     @property
-    def get_image_urls(self):
-        return self._get_stream_data_image_urls(self.home_featured_content.raw_data)
+    def offline_urls(self):
+        return [self.url] + collect_urls_from_streamfield(self.home_featured_content)
 
 
 class FeaturedContent(Orderable):
@@ -262,18 +266,16 @@ def get_progress_bar_eligible_sections():
         return Section.objects.exclude(pk__in=all_descendants)
 
     @property
-    def get_image_urls(self):
-        image_urls = []
+    def offline_urls(self):
+        urls = [self.url] + collect_urls_from_streamfield(self.body)
 
         if self.lead_image:
-            image_urls += self._get_renditions(self.lead_image)
+            urls += get_all_renditions_urls(self.lead_image)
 
         if self.image_icon:
-            image_urls += self._get_renditions(self.image_icon)
-
-        image_urls += self._get_stream_data_image_urls(self.body.raw_data)
+            urls += get_all_renditions_urls(self.image_icon)
 
-        return image_urls
+        return urls
 
     class Meta:
         verbose_name = _("section")
@@ -405,18 +407,16 @@ def top_level_section(self):
         return self.get_ancestors().filter(depth=4).first().specific
 
     @property
-    def get_image_urls(self):
-        image_urls = []
+    def offline_urls(self):
+        urls = [self.url] + collect_urls_from_streamfield(self.body)
 
         if self.lead_image:
-            image_urls += self._get_renditions(self.lead_image)
+            urls += get_all_renditions_urls(self.lead_image)
 
         if self.image_icon:
-            image_urls += self._get_renditions(self.image_icon)
-
-        image_urls += self._get_stream_data_image_urls(self.body.raw_data)
+            urls += get_all_renditions_urls(self.image_icon)
 
-        return image_urls
+        return urls
 
     class Meta:
         abstract = True
@@ -505,13 +505,8 @@ class BannerPage(Page, PageUtilsMixin):
     ]
 
     @property
-    def get_image_urls(self):
-        image_urls = []
-
-        if self.banner_image:
-            image_urls += self._get_renditions(self.banner_image)
-
-        return image_urls
+    def offline_urls(self):
+        return get_all_renditions_urls(self.banner_image)
 
 
 class FooterIndexPage(Page):

home/utils/__init__.py

@@ -0,0 +1,33 @@
+from bs4 import BeautifulSoup
+
+
+def get_all_renditions_urls(image):
+    return [
+        rendition.url
+        for rendition
+        in image.get_rendition_model().objects.filter(image_id=image.id)
+    ]
+
+
+def extract_urls_from_rich_text(text):
+    return [
+        img['src']
+        for img
+        in BeautifulSoup(str(text), 'lxml').find_all('img')
+    ]
+
+
+def collect_urls_from_streamfield(field):
+    urls = []
+    for block in field:
+        if block.block_type == 'image':
+            urls += get_all_renditions_urls(block.value)
+        if block.block_type == 'paragraph':
+            urls += extract_urls_from_rich_text(block.value)
+        if block.block_type == 'download':
+            urls += extract_urls_from_rich_text(
+                block.value.get('description', '')
+            )
+        if block.block_type == 'media':
+            urls.append(block.value.url)
+    return urls

iogt/tests.py

@@ -4,17 +4,18 @@
 from django.urls import reverse
 from rest_framework import status
 from wagtail.core.models import Site
+from wagtail.core.rich_text import RichText
 from wagtail_factories import ImageFactory, SiteFactory
-from wagtail.images.models import Image
 
 from home.factories import (
-    SectionFactory,
     ArticleFactory,
     HomePageFactory,
-    SVGToPNGMapFactory,
     LocaleFactory,
+    MediaFactory,
+    MiscellaneousIndexPageFactory,
     OfflineContentIndexPageFactory,
-    MiscellaneousIndexPageFactory
+    SectionFactory,
+    SVGToPNGMapFactory,
 )
 from iogt.utils import has_md5_hash
 
@@ -33,12 +34,11 @@ def setUp(self):
         self.en_offline_content_index_page = OfflineContentIndexPageFactory(parent=self.en_miscellaneous_index_page)
         self.ar_offline_content_index_page = OfflineContentIndexPageFactory(parent=self.ar_miscellaneous_index_page)
         self.section = SectionFactory(parent=self.en_home_page)
-        self.article = ArticleFactory(parent=self.en_home_page, body=[("image", ImageFactory())])
-        self.section_lead_image_rendition = self.section.lead_image.get_rendition('fill-360x360')
-        self.article_lead_image_rendition = self.article.lead_image.get_rendition('fill-360x360')
-        self.article_body_image_rendition = Image.objects.get(
-            id=self.article.body.raw_data[0]['value']).get_rendition('fill-360x360')
-        self.svg_to_png_map = SVGToPNGMapFactory()
+        self.article = ArticleFactory(
+            parent=self.en_home_page,
+            body=[("image", ImageFactory()),
+                  ("media", MediaFactory(type='video')),
+                  ("media", MediaFactory(type='audio'))])
 
     def test_root_page_is_returned(self):
         response = self.client.get(reverse(self.url_name, kwargs={'page_id': self.en_home_page.id}))
@@ -54,13 +54,45 @@ def test_descendants_are_returned(self):
         self.assertIn(self.article.url, response.data)
 
     def test_images_are_returned(self):
-        response = self.client.get(reverse(self.url_name, kwargs={'page_id': self.en_home_page.id}))
+        svg_to_png_map = SVGToPNGMapFactory()
+        section_lead_image = get_rendition(self.section.lead_image)
+        article_lead_image = get_rendition(self.article.lead_image)
+        body_image = get_rendition(self.article.body[0].value)
+
+        response = self.client.get(
+            reverse(self.url_name, kwargs={'page_id': self.en_home_page.id})
+        )
 
         self.assertEqual(response.status_code, status.HTTP_200_OK)
-        self.assertIn(self.section_lead_image_rendition.url, response.data)
-        self.assertIn(self.article_lead_image_rendition.url, response.data)
-        self.assertIn(self.article_body_image_rendition.url, response.data)
-        self.assertIn(self.svg_to_png_map.url, response.data)
+        self.assertIn(section_lead_image.url, response.data)
+        self.assertIn(article_lead_image.url, response.data)
+        self.assertIn(body_image.url, response.data)
+        self.assertIn(svg_to_png_map.url, response.data)
+
+    def test_media_are_returned(self):
+        response = self.client.get(
+            reverse(self.url_name, kwargs={'page_id': self.en_home_page.id})
+        )
+
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+        self.assertIn(self.article.body[1].value.url, response.data)
+        self.assertIn(self.article.body[2].value.url, response.data)
+
+    def test_image_urls_extracted_from_rich_text(self):
+        image = ImageFactory()
+        source = f'<embed embedtype="image" format="left" id="{image.id}"/>'
+        ArticleFactory(
+            parent=self.en_home_page,
+            body=[('paragraph', RichText(source))]
+        )
+        response = self.client.get(
+            reverse(self.url_name, kwargs={'page_id': self.en_home_page.id})
+        )
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+        renditions = image.get_rendition_model().objects.filter(image_id=image.id)
+        self.assertEquals(len(renditions), 1)
+        self.assertIn(renditions[0].url, response.data)
 
     def test_static_assets_are_returned(self):
         response = self.client.get(reverse(self.url_name, kwargs={'page_id': self.en_home_page.id}))
@@ -110,3 +142,7 @@ def test_has_md5_hash_without_has_values(self):
 
         for value in without_md5_hash:
             self.assertFalse(has_md5_hash(value))
+
+
+def get_rendition(image):
+    return image.get_rendition('fill-360x360')

iogt/views.py

@@ -120,12 +120,22 @@ def get(self, request, page_id):
             if page:
                 page_urls.append(page.url)
         translation.activate(active_locale.language_code)
-        image_urls = []
+
+        no_impl = set()
         for page in pages:
-            if isinstance(page, (HomePage, Section, Article, Poll, Survey, Quiz)):
-                page_urls.append(page.url)
-                image_urls += page.get_image_urls
+            try:
+                page_urls += page.offline_urls
+            except AttributeError:
+                no_impl.add(type(page).__name__)
+
+        if no_impl:
+            types = ", ".join(sorted(no_impl))
+            logger.warn(
+                f"Offline URLs could not be obtained for the following Page types: "
+                f"{types}"
+            )
 
+        image_urls = []
         for svg_to_png_map in SVGToPNGMap.objects.all():
             image_urls.append(svg_to_png_map.url)
 

questionnaires/models.py

@@ -17,8 +17,17 @@
 from wagtailsvg.edit_handlers import SvgChooserPanel
 from wagtailsvg.models import Svg
 
-from home.blocks import MediaBlock, PageButtonBlock, NumberedListBlock, RawHTMLBlock
+from home.blocks import (
+    MediaBlock,
+    NumberedListBlock,
+    PageButtonBlock,
+    RawHTMLBlock,
+)
 from home.mixins import PageUtilsMixin, TitleIconMixin
+from home.utils import (
+    collect_urls_from_streamfield,
+    get_all_renditions_urls,
+)
 from modelcluster.fields import ParentalKey
 from wagtail.admin.edit_handlers import (FieldPanel, InlinePanel,
                                          MultiFieldPanel, StreamFieldPanel)
@@ -235,8 +244,6 @@ def serve_questions_separately(self, request, *args, **kwargs):
         return render(request, self.template, context)
 
     def process_form_submission(self, form):
-        from home.models import SiteSettings
-
         user = form.user
         self.get_submission_class().objects.create(
             form_data=json.dumps(form.cleaned_data, cls=DjangoJSONEncoder),
@@ -270,16 +277,17 @@ def get_data_fields(self):
         return data_fields
 
     @property
-    def get_image_urls(self):
-        image_urls = []
+    def offline_urls(self):
+        urls = (
+            [self.url]
+            + collect_urls_from_streamfield(self.description)
+            + collect_urls_from_streamfield(self.thank_you_text)
+        )
 
         if self.image_icon:
-            image_urls += self._get_renditions(self.image_icon)
-
-        image_urls += self._get_stream_data_image_urls(self.description.raw_data)
-        image_urls += self._get_stream_data_image_urls(self.thank_you_text.raw_data)
+            urls += get_all_renditions_urls(self.image_icon)
 
-        return image_urls
+        return urls
 
     def get_submit_button_text(self, fields_step=None):
         submit_button_text = self.submit_button_text
@@ -633,7 +641,7 @@ def get_results(self, data=None):
         results = dict()
         results_list = []
         field = self.get_form_fields().first()
-        name, label, choices = field.clean_name, field.label, field.choices
+        name, choices = field.clean_name, field.choices
         submissions = self.get_submission_class().objects.filter(page=self)
         submissions = [submission for submission in submissions if submission.get_data().get(name) != '']