Merge pull request #98 from openzim/api-updates

API updates

Author: rgaudin

CHANGELOG.md

@@ -5,6 +5,37 @@ All notable changes to this project are documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) (as of version 1.5.0).
 
+## [Unreleased]
+
+⚠️ Warning: this release introduce several API changes to `zim.creator.Creator` and `zim.filesystem.make_zim_file`
+
+### Added
+
+- `zim.creator.Creator.config_metadata` method (returning Self) exposing all mandatory Metdata, all standard ones and allowing extra text metdadata.
+- `zim.creator.Creator.config_dev_metadata` method setting stub metdata for all mandatory ones (allowing overrides)
+- `zim.metadata` module with a list of per-metadata validation functions
+- `zim.creator.Creator.validate_metadata` (called on `start`) to verify metadata respects the spec (and its recommendations)
+- `zim.filesystem.make_zim_file` accepts a new optional `long_description` param.
+- `i18n.is_valid_iso_639_3` to check ISO-639-3 codes
+- `image.probing.is_valid_image` to check Image format and size
+
+### Changed
+
+- `zim.creator.Creator` `main_path` argument now mandatory
+- `zim.creator.Creator.start` now fails on missing required or invalid metadata
+- `zim.creator.Creator.add_metadata` nows enforces validation checks
+- `zim.filesystem.make_zim_file` renamed its `favicon_path` param to `illustration_path`
+- `zim.creator.Creator.config_indexing` `language` argument now optionnal when `indexing=False`
+- `zim.creator.Creator.config_indexing` now validates `language` is ISO- 639-3 when `indexing=True`
+
+### Removed
+
+- `zim.creator.Creator.update_metadata`. See `.config_metadata()` instead
+- `zim.creator.Creator` `language` argument. See `.config_metadata()` instead
+- `zim.creator.Creator` keyword arguments. See `.config_metadata()` instead
+- `zim.creator.Creator.add_default_illustration`. See `.config_metadata()` instead
+- `zim.archibe.Archive.media_counter` (deprecated in `2.0.0`)
+
 ## [2.1.0] - 2023-03-06
 
 ## Added

requirements.txt

@@ -4,7 +4,7 @@ colorthief==0.2.1
 python-resize-image>=1.1.19,<1.2
 Babel>=2.9,<3.0
 file-magic>=0.4.0,<0.5
-libzim>=2.1.0,<3.0
+libzim>=3.0.0,<3.1
 beautifulsoup4>=4.9.3,<4.10
 lxml>=4.6.3,<4.10
 optimize-images>=1.3.6,<1.6

src/zimscraperlib/constants.py

@@ -4,6 +4,7 @@
 
 import base64
 import pathlib
+import re
 
 ROOT_DIR = pathlib.Path(__file__).parent
 NAME = pathlib.Path(__file__).parent.name
@@ -39,9 +40,17 @@
     "Date": "2023-01-01",
     "Description": "Test Description",
     "Language": "fra",
+    # blank 48x48 transparent PNG
     "Illustration_48x48_at_1": base64.b64decode(
-        "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAAGXRFWHRTb2Z0d2FyZQBB"
-        "ZG9iZSBJbWFnZVJlYWR5ccllPAAAAA9JREFUeNpi+P//P0CAAQAF/gL+Lc6J7gAAAABJ"
-        "RU5ErkJggg=="
+        "iVBORw0KGgoAAAANSUhEUgAAADAAAAAwAQMAAABtzGvEAAAAGXRFWHRTb2Z0d2FyZQBB"
+        "ZG9iZSBJbWFnZVJlYWR5ccllPAAAAANQTFRFR3BMgvrS0gAAAAF0Uk5TAEDm2GYAAAAN"
+        "SURBVBjTY2AYBdQEAAFQAAGn4toWAAAAAElFTkSuQmCC"
     ),
 }
+
+MAXIMUM_DESCRIPTION_METADATA_LENGTH = 80
+MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH = 4000
+
+ILLUSTRATIONS_METADATA_RE = re.compile(
+    r"^Illustration_(?P<height>\d+)x(?P<width>\d+)@(?P<scale>\d+)$"
+)

src/zimscraperlib/i18n.py

@@ -179,3 +179,8 @@ def get_language_details(query: str, failsafe: Optional[bool] = False) -> Dict:
         }
     )
     return iso_data
+
+
+def is_valid_iso_639_3(code: str) -> bool:
+    """whether code is a valid ISO-639-3 code"""
+    return (get_language_details(code, failsafe=True) or {}).get("iso-639-3") == code

src/zimscraperlib/image/__init__.py

@@ -5,6 +5,7 @@
 # flake8: noqa
 from .convertion import convert_image
 from .optimization import optimize_image
+from .probing import is_valid_image
 from .transformation import resize_image
 
-__all__ = ["convert_image", "optimize_image", "resize_image"]
+__all__ = ["convert_image", "is_valid_image", "optimize_image", "resize_image"]

src/zimscraperlib/image/probing.py

@@ -60,3 +60,22 @@ def format_for(src: Union[pathlib.Path, io.BytesIO], from_suffix: bool = True) -
 
     init_pil()
     return ext_fmt_map[src.suffix]  # might raise KeyError on unknown extension
+
+
+def is_valid_image(
+    image: Union[pathlib.Path, io.IOBase, bytes],
+    imformat: str,
+    size: Optional[Tuple[int, int]] = None,
+) -> bool:
+    """whether image is a valid imformat (PNG) image, optionnaly of requested size"""
+    if isinstance(image, bytes):
+        image = io.BytesIO(image)
+    try:
+        img = PIL.Image.open(image)
+        if img.format != imformat:
+            return False
+        if size and img.size != size:
+            return False
+    except Exception:
+        return False
+    return True

src/zimscraperlib/zim/archive.py

@@ -10,7 +10,6 @@
     - direct access to search results and number of results
     - public Entry access by Id"""
 
-import warnings
 from typing import Dict, Iterable, List, Optional
 
 import libzim.reader
@@ -107,20 +106,3 @@ def counters(self) -> Dict[str, int]:
             return parseMimetypeCounter(self.get_text_metadata("Counter"))
         except RuntimeError:  # pragma: no cover (no ZIM avail to test itl)
             return {}  # pragma: no cover
-
-    @property
-    def article_counter(self) -> int:
-        warnings.warn(
-            "Archive.article_counter now deprecated. "
-            "Use Archive.article_count instead",
-            DeprecationWarning,
-        )
-        return self.article_count
-
-    @property
-    def media_counter(self) -> int:
-        warnings.warn(
-            "Archive.media_counter now deprecated. " "Use Archive.media_count instead",
-            DeprecationWarning,
-        )
-        return self.media_count

src/zimscraperlib/zim/creator.py

@@ -32,8 +32,21 @@
     MANDATORY_ZIM_METADATA_KEYS,
 )
 from ..filesystem import delete_callback, get_content_mimetype, get_file_mimetype
+from ..i18n import is_valid_iso_639_3
 from ..types import get_mime_for_name
 from .items import StaticItem
+from .metadata import (
+    validate_counter,
+    validate_date,
+    validate_description,
+    validate_illustrations,
+    validate_language,
+    validate_longdescription,
+    validate_required_values,
+    validate_standard_str_types,
+    validate_tags,
+    validate_title,
+)
 
 DUPLICATE_EXC_STR = re.compile(
     r"^Impossible to add(.+)"
@@ -91,6 +104,7 @@ def __init__(
     ):
         super().__init__(filename=filename)
         self._metadata = dict()
+        self.__indexing_configured = False
         self.can_finish = True
 
         self.set_mainpath(main_path)
@@ -105,18 +119,28 @@ def __init__(
         self.workaround_nocancel = workaround_nocancel
         self.ignore_duplicates = ignore_duplicates
 
+    def config_indexing(self, indexing: bool, language: Optional[str] = None):
+        """Toggle full-text and title indexing of entries
+
+        Uses Language metadata's value (or "") if not set"""
+        language = language or self._metadata.get("Language", "")
+        if indexing and not is_valid_iso_639_3(language):
+            raise ValueError("Not a valid ISO-639-3 language code")
+        super().config_indexing(indexing, language)
+        self.__indexing_configured = True
+        return self
+
     def start(self):
-        if not all(
-            [
-                key in self._metadata.keys() and self._metadata.get(key, None)
-                for key in MANDATORY_ZIM_METADATA_KEYS
-            ]
-        ):
+        if not all([self._metadata.get(key) for key in MANDATORY_ZIM_METADATA_KEYS]):
             raise ValueError("Mandatory metadata are not all set.")
 
         for name, value in self._metadata.items():
             if value:
-                self._validate_metadata(name, value)
+                self.validate_metadata(name, value)
+
+        language = self._metadata.get("Language", "").split(",")
+        if language[0] and not self.__indexing_configured:
+            self.config_indexing(True, language[0])
 
         super().__enter__()
 
@@ -128,15 +152,36 @@ def start(self):
 
         return self
 
-    def _validate_metadata(self, name, value):
-        if name == "Counter":
-            raise ValueError("You do not need to set Counter.")
+    def validate_metadata(
+        self,
+        name: str,
+        value: Union[bytes, str, datetime.datetime, datetime.date, Iterable[str]],
+    ):
+        """Ensures metadata value for name is conform with the openZIM spec on Metadata
+
+        Also enforces recommendations
+        See https://wiki.openzim.org/wiki/Metadata"""
 
-        if name == "Description" and len(value) > 80:
-            raise ValueError("Description is too long.")
+        validate_required_values(name, value)
+        validate_standard_str_types(name, value)
 
-        if name == "LongDescription" and len(value) > 4000:
-            raise ValueError("LongDescription is too long.")
+        validate_title(name, value)
+        validate_date(name, value)
+        validate_language(name, value)
+        validate_counter(name, value)
+        validate_description(name, value)
+        validate_longdescription(name, value)
+        validate_tags(name, value)
+        validate_illustrations(name, value)
+
+    def add_metadata(
+        self,
+        name: str,
+        content: Union[str, bytes, datetime.date, datetime.datetime],
+        mimetype: str = "text/plain;charset=UTF-8",
+    ):
+        self.validate_metadata(name, content)
+        super().add_metadata(name, content, mimetype)
 
     def config_metadata(
         self,
@@ -158,17 +203,7 @@ def config_metadata(
         Relation: Optional[str] = None,
         **extras: str,
     ):
-        """
-        A chaining functions which configures the metadata of the Creator class.
-        You must set all mandatory metadata in this phase.
-
-        Parameters:
-            check out: https://wiki.openzim.org/wiki/Metadata
-            all the extra metadata must be plain text.
-
-        Returns:
-            Self
-        """
+        """Sets all mandatory Metadata as well as standard and any other text ones"""
         self._metadata.update(
             {
                 "Name": Name,
@@ -189,18 +224,10 @@ def config_metadata(
             }
         )
         self._metadata.update(extras)
-        language = self._metadata.get("Language", "").split(",")
-        self.config_indexing(True, language[0])
-
         return self
 
     def config_dev_metadata(self, **extras: str):
-        """
-        A Test function. It will set the default test metadata for a Creator instance.
-
-        Returns:
-            Self
-        """
+        """Calls config_metadata with default (yet overridable) values for dev"""
         devel_default_metadata = DEFAULT_DEV_ZIM_METADATA.copy()
         devel_default_metadata.update(extras)
         return self.config_metadata(**devel_default_metadata)

src/zimscraperlib/zim/filesystem.py

@@ -114,7 +114,7 @@ def make_zim_file(
     fpath: pathlib.Path,
     name: str,
     main_page: str,
-    favicon: str,
+    illustration: str,
     title: str,
     description: str,
     date: datetime.date = None,
@@ -135,7 +135,7 @@ def make_zim_file(
     """Creates a zimwriterfs-like ZIM file at {fpath} from {build_dir}
 
     main_page: path of item to serve as main page
-    favicon: relative path to favicon file in build_dir
+    illustration: relative path to illustration file in build_dir
     tags: list of str tags to add to meta
     redirects: list of (src, dst, title) tuple to create redirects from
     rewrite_links controls whether to rewrite HTML/CSS content
@@ -146,12 +146,12 @@ def make_zim_file(
     if not build_dir.exists() or not build_dir.is_dir():
         raise IOError(f"Incorrect build_dir: {build_dir}")
 
-    favicon_path = build_dir / favicon
-    if not favicon_path.exists() or not favicon_path.is_file():
-        raise IOError(f"Incorrect favicon: {favicon} ({favicon_path})")
+    illustration_path = build_dir / illustration
+    if not illustration_path.exists() or not illustration_path.is_file():
+        raise IOError(f"Incorrect illustration: {illustration} ({illustration_path})")
 
-    with open(favicon_path, "rb") as fh:
-        favicon_data = fh.read()
+    with open(illustration_path, "rb") as fh:
+        illustration_data = fh.read()
 
     zim_file = Creator(filename=fpath, main_path=main_page).config_metadata(
         **{
@@ -171,7 +171,7 @@ def make_zim_file(
                 "Flavour": flavour,
                 "Scraper": scraper,
                 "LongDescription": long_description,
-                "Illustration_48x48_at_1": favicon_data,
+                "Illustration_48x48_at_1": illustration_data,
             }.items()
             if v is not None
         }