Merge pull request #147 from openzim/scraperlib_3.3.2

Add arguments to make_zim_file and relax dependencies constraints

Author: benoit74

.pre-commit-config.yaml

@@ -7,15 +7,15 @@ repos:
   -   id: trailing-whitespace
   -   id: end-of-file-fixer
 - repo: https://github.com/psf/black
-  rev: "24.1.1"
+  rev: "24.3.0"
   hooks:
   -   id: black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.2.1
+  rev: v0.3.3
   hooks:
   - id: ruff
 - repo: https://github.com/RobertCraigie/pyright-python
-  rev: v1.1.350
+  rev: v1.1.354
   hooks:
   - id: pyright
     name: pyright (system)

CHANGELOG.md

@@ -7,6 +7,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Add support for `disable_metadata_checks` and `ignore_duplicates` arguments in `make_zim_file` function ("zimwritefs-mode")
+
+### Changed
+
+- Relaxed constraints on Python dependencies
+- Upgraded optional dependencies used for test and QA
+
 ## [3.3.1] - 2024-02-27
 
 ### Added

pyproject.toml

@@ -15,9 +15,9 @@ dependencies = [
   "Babel>=2.9,<3.0",
   "python-magic>=0.4.3,<0.5",
   "libzim>=3.4.0,<4.0",
-  "beautifulsoup4>=4.9.3,<4.10", # upgrade to 4.10 and later to be done
-  "lxml>=4.6.3,<4.10", # upgrade to 4.10 and later to be done
-  "optimize-images>=1.3.6,<1.6",
+  "beautifulsoup4>=4.9.3,<5.0",
+  "lxml>=4.6.3,<6.0",
+  "optimize-images>=1.3.6,<2.0",
   # youtube-dl should be updated as frequently as possible
   "yt-dlp"
 ]
@@ -36,20 +36,20 @@ scripts = [
   "invoke==2.2.0",
 ]
 lint = [
-  "black==24.1.1",
-  "ruff==0.2.1",
+  "black==24.3.0",
+  "ruff==0.3.3",
 ]
 check = [
-  "pyright==1.1.350",
+  "pyright==1.1.354",
 ]
 test = [
-  "pytest==8.0.0",
+  "pytest==8.1.1",
   "pytest-mock==3.12.0",
-  "coverage==7.4.1",
+  "coverage==7.4.4",
 ]
 dev = [
-  "pre-commit==3.5.0",
-  "debugpy==1.8.0",
+  "pre-commit==3.6.2",
+  "debugpy==1.8.1",
   "zimscraperlib[scripts]",
   "zimscraperlib[lint]",
   "zimscraperlib[test]",

src/zimscraperlib/zim/filesystem.py

@@ -135,6 +135,8 @@ def make_zim_file(
     redirects_file: pathlib.Path = None,  # noqa: RUF013  # pyright: ignore
     rewrite_links: bool = True,  # noqa: FBT001, FBT002, ARG001
     workaround_nocancel: bool = True,  # noqa: FBT001, FBT002
+    ignore_duplicates: bool = True,  # noqa: FBT001, FBT002
+    disable_metadata_checks: bool = False,  # noqa: FBT001, FBT002
 ):
     """Creates a zimwriterfs-like ZIM file at {fpath} from {build_dir}
 
@@ -157,7 +159,12 @@ def make_zim_file(
     with open(illustration_path, "rb") as fh:
         illustration_data = fh.read()
 
-    zim_file = Creator(filename=fpath, main_path=main_page).config_metadata(
+    zim_file = Creator(
+        filename=fpath,
+        main_path=main_page,
+        ignore_duplicates=ignore_duplicates,
+        disable_metadata_checks=disable_metadata_checks,
+    ).config_metadata(
         **{
             k: v
             for k, v in {