Improvements to Creator API

- Requiring libzim 3.0 to support setting LongDescription metadata
- Exposing Creator.validate_metadata
- Expanded validate_metadata to support all current checks for mandatory and recommendations
- Added tests for all validate_metadata cases
- Exposing max length for (Long)Description so scrapers can use it in their entrypoints
- Marking `Creator.config_indexing`'s `language` argument optional (when disabling)
- Only indexing on start() if config_indexing() was not already called manually
- Fixed default dev illustration to be 48x48
- Added new image format and size validation check in image.probing module
- Fixed test image (commons.png) to be 48x48
- Adapted tests for commons48

Author: rgaudin

CHANGELOG.md

@@ -5,6 +5,36 @@ All notable changes to this project are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
 
+## [Unreleased]
+
+⚠️ Warning: this release introduce several API changes to `zim.creator.Creator` and `zim.filesystem.make_zim_file`
+
+### Added
+
+- `zim.creator.Creator.config_metadata` method (returning Self) exposing all mandatory Metdata, all standard ones and allowing extra text metdadata.
+- `zim.creator.Creator.config_dev_metadata` method setting stub metdata for all mandatory ones (allowing overrides)
+- `zim.creator.Creator.validate_metadata` (called on `start`) to verify metadata respects the spec (and its recommendations)
+- `zim.filesystem.make_zim_file` accepts a new optional `long_description` param.
+- `i18n.is_valid_iso_639_3` to check ISO-639-3 codes
+- `image.probing.is_valid_image` to check Image format and size
+
+### Changed
+
+- `zim.creator.Creator` `main_path` argument now mandatory
+- `zim.creator.Creator.start` now fails on missing required or invalid metadata
+- `zim.creator.Creator.add_metadata` nows enforces validation checks
+- `zim.filesystem.make_zim_file` renamed its `favicon_path` param to `illustration_path`
+- `zim.creator.Creator.config_indexing` `language` argument now optionnal when `indexing=False`
+- `zim.creator.Creator.config_indexing` now validates `language` is ISO- 639-3 when `indexing=True`
+
+### Removed
+
+- `zim.creator.Creator.update_metadata`. See `.config_metadata()` instead
+- `zim.creator.Creator` `language` argument. See `.config_metadata()` instead
+- `zim.creator.Creator` keyword arguments. See `.config_metadata()` instead
+- `zim.creator.Creator.add_default_illustration`. See `.config_metadata()` instead
+- `zim.archibe.Archive.media_counter` (deprecated in `2.0.0`)
+
 ## [2.1.0] - 2023-03-06
 
 ## Added

requirements.txt

@@ -4,7 +4,7 @@ colorthief==0.2.1
 python-resize-image>=1.1.19,<1.2
 Babel>=2.9,<3.0
 file-magic>=0.4.0,<0.5
-libzim>=2.1.0,<3.0
+libzim>=3.0.0,<3.1
 beautifulsoup4>=4.9.3,<4.10
 lxml>=4.6.3,<4.10
 optimize-images>=1.3.6,<1.6

src/zimscraperlib/constants.py

@@ -4,6 +4,7 @@
 
 import base64
 import pathlib
+import re
 
 ROOT_DIR = pathlib.Path(__file__).parent
 NAME = pathlib.Path(__file__).parent.name
@@ -39,9 +40,17 @@
     "Date": "2023-01-01",
     "Description": "Test Description",
     "Language": "fra",
+    # blank 48x48 transparent PNG
     "Illustration_48x48_at_1": base64.b64decode(
-        "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAAGXRFWHRTb2Z0d2FyZQBB"
-        "ZG9iZSBJbWFnZVJlYWR5ccllPAAAAA9JREFUeNpi+P//P0CAAQAF/gL+Lc6J7gAAAABJ"
-        "RU5ErkJggg=="
+        "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwAQMAAABtzGvEAAAAGXRFWHRTb2Z0d2FyZQBB"
+        "ZG9iZSBJbWFnZVJlYWR5ccllPAAAAANQTFRFR3BMgvrS0gAAAAF0Uk5TAEDm2GYAAAAN"
+        "SURBVBjTY2AYBdQEAAFQAAGn4toWAAAAAElFTkSuQmCC"
     ),
 }
+
+MAXIMUM_DESCRIPTION_METADATA_LENGTH = 80
+MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH = 4000
+
+ILLUSTRATIONS_METADATA_RE = re.compile(
+    r"^Illustration_(?P<height>\d+)x(?P<width>\d+)@(?P<scale>\d+)$"
+)

src/zimscraperlib/image/__init__.py

@@ -5,6 +5,7 @@
 # flake8: noqa
 from .convertion import convert_image
 from .optimization import optimize_image
+from .probing import is_valid_image
 from .transformation import resize_image
 
-__all__ = ["convert_image", "optimize_image", "resize_image"]
+__all__ = ["convert_image", "is_valid_image", "optimize_image", "resize_image"]

src/zimscraperlib/image/probing.py

@@ -60,3 +60,22 @@ def format_for(src: Union[pathlib.Path, io.BytesIO], from_suffix: bool = True) -
 
     init_pil()
     return ext_fmt_map[src.suffix]  # might raise KeyError on unknown extension
+
+
+def is_valid_image(
+    image: Union[pathlib.Path, io.IOBase, bytes],
+    imformat: str,
+    size: Optional[Tuple[int, int]] = None,
+) -> bool:
+    """whether image is a valid imformat (PNG) image, optionnaly of requested size"""
+    if isinstance(image, bytes):
+        image = io.BytesIO(image)
+    try:
+        img = PIL.Image.open(image)
+        if img.format != imformat:
+            return False
+        if size and img.size != size:
+            return False
+    except Exception:
+        return False
+    return True

src/zimscraperlib/zim/creator.py

@@ -22,16 +22,22 @@
 import pathlib
 import re
 import weakref
+from collections.abc import Iterable as IterableT
 from typing import Any, Callable, Iterable, Optional, Tuple, Union
 
 import libzim.writer
 
 from ..constants import (
     DEFAULT_DEV_ZIM_METADATA,
     FRONT_ARTICLE_MIMETYPES,
+    ILLUSTRATIONS_METADATA_RE,
     MANDATORY_ZIM_METADATA_KEYS,
+    MAXIMUM_DESCRIPTION_METADATA_LENGTH,
+    MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH,
 )
 from ..filesystem import delete_callback, get_content_mimetype, get_file_mimetype
+from ..i18n import is_valid_iso_639_3
+from ..image.probing import is_valid_image
 from ..types import get_mime_for_name
 from .items import StaticItem
 
@@ -91,6 +97,7 @@ def __init__(
     ):
         super().__init__(filename=filename)
         self._metadata = dict()
+        self.__indexing_configured = False
         self.can_finish = True
 
         self.set_mainpath(main_path)
@@ -105,18 +112,28 @@ def __init__(
         self.workaround_nocancel = workaround_nocancel
         self.ignore_duplicates = ignore_duplicates
 
+    def config_indexing(self, indexing: bool, language: Optional[str] = None):
+        """Toggle full-text and title indexing of entries
+
+        Uses Language metadata's value (or "") if not set"""
+        language = language or self._metadata.get("Language", "")
+        if indexing and not is_valid_iso_639_3(language):
+            raise ValueError("Not a valid ISO-639-3 language code")
+        super().config_indexing(indexing, language)
+        self.__indexing_configured = True
+        return self
+
     def start(self):
-        if not all(
-            [
-                key in self._metadata.keys() and self._metadata.get(key, None)
-                for key in MANDATORY_ZIM_METADATA_KEYS
-            ]
-        ):
+        if not all([self._metadata.get(key) for key in MANDATORY_ZIM_METADATA_KEYS]):
             raise ValueError("Mandatory metadata are not all set.")
 
         for name, value in self._metadata.items():
             if value:
-                self._validate_metadata(name, value)
+                self.validate_metadata(name, value)
+
+        language = self._metadata.get("Language", "").split(",")
+        if language[0] and not self.__indexing_configured:
+            self.config_indexing(True, language[0])
 
         super().__enter__()
 
@@ -128,15 +145,97 @@ def start(self):
 
         return self
 
-    def _validate_metadata(self, name, value):
+    def validate_metadata(
+        self,
+        name: str,
+        value: Union[bytes, str, datetime.datetime, datetime.date, Iterable[str]],
+    ):
+        """Ensures metadata value for name is conform with the openZIM spec on Metadata
+
+        Also enforces recommendations
+        See https://wiki.openzim.org/wiki/Metadata"""
+
+        # spec doesnt require any value but empty strings are not useful
+        if name in MANDATORY_ZIM_METADATA_KEYS and not value:
+            raise ValueError(f"Missing value for {name}")
+
+        # most require/standard and al
+        if name in (
+            "Name",
+            "Title",
+            "Creator",
+            "Publisher",
+            "Description",
+            "LongDescription",
+            "License",
+            "Relation",
+            "Relation",
+            "Flavour",
+            "Source",
+            "Scraper",
+        ) and not isinstance(value, str):
+            raise ValueError(f"Invalid type for {name}")
+
+        if name == "Title" and len(value) > 30:
+            raise ValueError(f"{name} is too long.")
+
+        if name == "Date":
+            if not isinstance(value, (datetime.datetime, datetime.date, str)):
+                raise ValueError(f"Invalid type for {name}.")
+            elif isinstance(value, str):
+                match = re.match(
+                    r"(?P<year>\d{4})-(?P<month>\d{2})-(?P<day>\d{2})", value
+                )
+                try:
+                    datetime.date(**{k: int(v) for k, v in match.groupdict().items()})
+                except Exception as exc:
+                    raise ValueError(f"Invalid {name} format: {exc}")
+
+        if name == "Language" and not is_valid_iso_639_3(value):
+            raise ValueError(f"{value} is not ISO-639-3.")
+
         if name == "Counter":
-            raise ValueError("You do not need to set Counter.")
+            raise ValueError(f"{name} cannot be set. libzim sets it.")
+
+        if name == "Description" and len(value) > MAXIMUM_DESCRIPTION_METADATA_LENGTH:
+            raise ValueError(f"{name} is too long.")
 
-        if name == "Description" and len(value) > 80:
-            raise ValueError("Description is too long.")
+        if (
+            name == "LongDescription"
+            and len(value) > MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH
+        ):
+            raise ValueError(f"{name} is too long.")
 
-        if name == "LongDescription" and len(value) > 4000:
-            raise ValueError("LongDescription is too long.")
+        if name == "Tags" and (
+            not isinstance(value, IterableT)
+            or not all([isinstance(tag, str) for tag in value])
+        ):
+            raise ValueError(f"Invalid type(s) for {name}")
+
+        if name.startswith("Illustration_"):
+            match = ILLUSTRATIONS_METADATA_RE.match(name)
+            if match and not is_valid_image(
+                image=value,
+                imformat="PNG",
+                size=(
+                    int(match.groupdict()["width"]),
+                    int(match.groupdict()["height"]),
+                ),
+            ):
+                raise ValueError(
+                    f"{name} is not a "
+                    f"{match.groupdict()['width']}x{match.groupdict()['height']} "
+                    "PNG Image"
+                )
+
+    def add_metadata(
+        self,
+        name: str,
+        content: Union[str, bytes, datetime.date, datetime.datetime],
+        mimetype: str = "text/plain;charset=UTF-8",
+    ):
+        self.validate_metadata(name, content)
+        super().add_metadata(name, content, mimetype)
 
     def config_metadata(
         self,
@@ -158,17 +257,7 @@ def config_metadata(
         Relation: Optional[str] = None,
         **extras: str,
     ):
-        """
-        A chaining functions which configures the metadata of the Creator class.
-        You must set all mandatory metadata in this phase.
-
-        Parameters:
-            check out: https://wiki.openzim.org/wiki/Metadata
-            all the extra metadata must be plain text.
-
-        Returns:
-            Self
-        """
+        """Sets all mandatory Metadata as well as standard and any other text ones"""
         self._metadata.update(
             {
                 "Name": Name,
@@ -189,18 +278,10 @@ def config_metadata(
             }
         )
         self._metadata.update(extras)
-        language = self._metadata.get("Language", "").split(",")
-        self.config_indexing(True, language[0])
-
         return self
 
     def config_dev_metadata(self, **extras: str):
-        """
-        A Test function. It will set the default test metadata for a Creator instance.
-
-        Returns:
-            Self
-        """
+        """Calls config_metadata with default (yet overridable) values for dev"""
         devel_default_metadata = DEFAULT_DEV_ZIM_METADATA.copy()
         devel_default_metadata.update(extras)
         return self.config_metadata(**devel_default_metadata)

tests/conftest.py

@@ -85,6 +85,11 @@ def file_src(fname):
 
 @pytest.fixture(scope="module")
 def png_image():
+    return file_src("commons48.png")
+
+
+@pytest.fixture(scope="module")
+def png_image2():
     return file_src("commons.png")