check: support custom archive formatting, #9411

docs/usage/general/environment.rst.inc

@@ -142,6 +142,8 @@ General:
             Now you can init a fresh repo. Make sure you do not use the workaround any more.
 
 Output formatting:
+    BORG_CHECK_FORMAT
+        Giving the default value for ``borg check --format=X``.
     BORG_LIST_FORMAT
         Giving the default value for ``borg list --format=X``.
     BORG_REPO_LIST_FORMAT

src/borg/archive.py

@@ -35,6 +35,7 @@
 from .platform import uid2user, user2uid, gid2group, group2gid, get_birthtime_ns
 from .helpers import parse_timestamp, archive_ts_now, CompressionSpec
 from .helpers import OutputTimestamp, format_timedelta, format_file_size, file_status, FileSize
+from .helpers import ArchiveFormatter
 from .helpers import safe_encode, make_path_safe, remove_surrogates, text_to_json, join_cmd, remove_dotdot_prefixes
 from .helpers import StableDict
 from .helpers import bin_to_hex
@@ -1763,6 +1764,8 @@ def check(
         newer=None,
         oldest=None,
         newest=None,
+        format,
+        iec=False,
     ):
         """Perform a set of checks on 'repository'
 
@@ -1773,13 +1776,17 @@ def check(
         :param older/newer: only check archives older/newer than timedelta from now
         :param oldest/newest: only check archives older/newer than timedelta from oldest/newest archive timestamp
         :param verify_data: integrity verification of data referenced by archives
+        :param format: format string used to describe an archive in the log output
+        :param iec: format sizes using IEC units (1KiB = 1024B)
         """
         if not isinstance(repository, Repository):
             logger.error("Checking legacy repositories is not supported.")
             return False
         logger.info("Starting archive consistency check...")
         self.check_all = not any((first, last, match, older, newer, oldest, newest))
         self.repair = repair
+        self.format = format
+        self.iec = iec
         self.repository = repository
         # A normal (non-repair) archives check trusts the in-repo index: the repository check verified
         # each index object's sha256, and the index is the authoritative record of which chunks exist,
@@ -2139,16 +2146,21 @@ def valid_item(obj):
         else:
             archive_infos = self.manifest.archives.list(sort_by=sort_by)
         num_archives = len(archive_infos)
+        formatter = ArchiveFormatter(self.format, self.repository, self.manifest, self.key, iec=self.iec)
 
         pi = ProgressIndicatorPercent(
             total=num_archives, msg="Checking archives %3.1f%%", step=0.1, msgid="check.rebuild_archives"
         )
         for i, info in enumerate(archive_infos):
             pi.show(i)
             archive_id, archive_id_hex = info.id, bin_to_hex(info.id)
-            logger.info(
-                f"Analyzing archive {info.name} {info.ts.astimezone()} {archive_id_hex} ({i + 1}/{num_archives})"
-            )
+            try:
+                formatted = formatter.format_item(info, jsonline=False)
+            except (Archive.DoesNotExist, Repository.ObjectNotFound, IntegrityErrorBase):
+                # keys like {comment} need the archive metadata, which is damaged or missing here.
+                # use the values from the archive directory entry, they are always available.
+                formatted = f"{info.name} {OutputTimestamp(info.ts)} {archive_id_hex}"
+            logger.info(f"Analyzing archive {formatted} ({i + 1}/{num_archives})")
             if archive_id not in self.chunks:
                 logger.error(f"Archive metadata block {archive_id_hex} is missing!")
                 self.error_found = True

src/borg/archiver/check_cmd.py

@@ -1,8 +1,11 @@
+import os
+from string import Formatter
+
 from ._common import with_repository, Highlander
 from ..archive import ArchiveChecker
 from ..constants import *  # NOQA
 from ..helpers import set_ec, EXIT_WARNING, CancelledByUser, CommandError, IntegrityError
-from ..helpers import yes
+from ..helpers import yes, ArchiveFormatter
 from ..helpers.argparsing import ArgumentParser
 
 from ..logger import create_logger
@@ -31,9 +34,12 @@ def do_check(self, args, repository):
                 env_var_override="BORG_CHECK_I_KNOW_WHAT_I_AM_DOING",
             ):
                 raise CancelledByUser()
-        if args.repo_only and any((args.verify_data, args.first, args.last, args.match_archives)):
+        if args.repo_only and any(
+            (args.verify_data, args.first, args.last, args.match_archives, args.format is not None)
+        ):
             raise CommandError(
-                "--repository-only contradicts --first, --last, -a / --match-archives and --verify-data arguments."
+                "--repository-only contradicts --first, --last, -a / --match-archives, "
+                "--verify-data and --format arguments."
             )
         if args.repo_only and args.find_lost_archives:
             raise CommandError("--repository-only contradicts the --find-lost-archives option.")
@@ -51,6 +57,19 @@ def do_check(self, args, repository):
                 archive_checker.key = archive_checker.make_key(repository, manifest_only=True)
             except IntegrityError:
                 pass  # will try to make key later again
+            if args.format is not None:
+                format = args.format
+            else:
+                format = os.environ.get("BORG_CHECK_FORMAT", "{archive} {time} {id}")
+            # check the format here, the archives check formats the first archive only after
+            # the repository check has finished, which can take hours.
+            try:
+                format_keys = {key for _, key, _, _ in Formatter().parse(format) if key}
+            except ValueError as err:
+                raise CommandError(f"Invalid format string: {err}")
+            unknown_keys = format_keys - set(ArchiveFormatter.KEY_DESCRIPTIONS) - set(ArchiveFormatter.FIXED_KEYS)
+            if unknown_keys:
+                raise CommandError(f"Invalid format keys: {', '.join(sorted(unknown_keys))}")
         if not args.archives_only:
             if not repository.check(repair=args.repair, max_duration=args.max_duration):
                 set_ec(EXIT_WARNING)
@@ -67,6 +86,8 @@ def do_check(self, args, repository):
             newer=args.newer,
             oldest=args.oldest,
             newest=args.newest,
+            format=format,
+            iec=args.iec,
         ):
             set_ec(EXIT_WARNING)
             return
@@ -139,6 +160,10 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
         data from the repository and is thus very time-consuming. You cannot use
         ``--find-lost-archives`` with ``--repository-only``.
 
+        You can influence how the archive part of the ``Analyzing archive ...`` output is
+        formatted by giving a custom format using ``--format`` (see the ``borg repo-list``
+        description for more details about the format string).
+
         About repair mode
         +++++++++++++++++
 
@@ -210,4 +235,11 @@ def build_parser_check(self, subparsers, common_parser, mid_common_parser):
             action=Highlander,
             help="perform only a partial repository check for at most SECONDS seconds (default: unlimited)",
         )
+        subparser.add_argument(
+            "--format",
+            metavar="FORMAT",
+            dest="format",
+            action=Highlander,
+            help="specify format for the archive part " '(default: "{archive} {time} {id}")',
+        )
         define_archive_filters_group(subparser)

src/borg/testsuite/archiver/check_cmd_test.py

@@ -1,13 +1,14 @@
 from datetime import datetime, timezone, timedelta
 from pathlib import Path
+import re
 import shutil
 from unittest.mock import patch
 
 import pytest
 
 from ...archive import ChunkBuffer
 from ...constants import *  # NOQA
-from ...helpers import bin_to_hex, msgpack
+from ...helpers import bin_to_hex, msgpack, CommandError
 from ...manifest import Manifest
 from ...repository import Repository
 from ..repository_test import fchunk, corrupt_chunk_on_disk
@@ -215,6 +216,79 @@ def test_missing_archive_metadata(archivers, request):
     cmd(archiver, "check", exit_code=0)
 
 
+def test_check_format(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    check_cmd_setup(archiver)
+    output = cmd(archiver, "check", "-v", "--archives-only", "--format", "{archive}|{hostname}", exit_code=0)
+    assert "Analyzing archive archive1|" in output
+
+
+def test_check_format_env_var(archivers, request, monkeypatch):
+    archiver = request.getfixturevalue(archivers)
+    check_cmd_setup(archiver)
+    monkeypatch.setenv("BORG_CHECK_FORMAT", "{archive}|env")
+    output = cmd(archiver, "check", "-v", "--archives-only", exit_code=0)
+    assert "Analyzing archive archive1|env" in output
+    output = cmd(archiver, "check", "-v", "--archives-only", "--format", "{archive}|arg", exit_code=0)
+    assert "Analyzing archive archive1|arg" in output  # --format overrides the env var
+
+
+def test_check_format_invalid_key(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    check_cmd_setup(archiver)
+    if archiver.FORK_DEFAULT:
+        expected_ec = CommandError().exit_code
+        output = cmd(archiver, "check", "--archives-only", "--format", "{nosuchkey}", exit_code=expected_ec)
+        assert "Invalid format keys: nosuchkey" in output
+    else:
+        with pytest.raises(CommandError, match="Invalid format keys: nosuchkey"):
+            cmd(archiver, "check", "--archives-only", "--format", "{nosuchkey}")
+
+
+def test_check_format_repository_only(archivers, request, monkeypatch):
+    archiver = request.getfixturevalue(archivers)
+    check_cmd_setup(archiver)
+    if archiver.FORK_DEFAULT:
+        expected_ec = CommandError().exit_code
+        output = cmd(archiver, "check", "--repository-only", "--format", "{archive}", exit_code=expected_ec)
+        assert "--repository-only contradicts" in output
+    else:
+        with pytest.raises(CommandError, match="--repository-only contradicts"):
+            cmd(archiver, "check", "--repository-only", "--format", "{archive}")
+    # only the option contradicts, a set env var must not make the repository check fail:
+    monkeypatch.setenv("BORG_CHECK_FORMAT", "{archive}|env")
+    cmd(archiver, "check", "--repository-only", exit_code=0)
+
+
+def test_check_format_invalid_format_string(archivers, request):
+    archiver = request.getfixturevalue(archivers)
+    check_cmd_setup(archiver)
+    if archiver.FORK_DEFAULT:
+        expected_ec = CommandError().exit_code
+        output = cmd(archiver, "check", "--archives-only", "--format", "{archive", exit_code=expected_ec)
+        assert "Invalid format string" in output
+    else:
+        with pytest.raises(CommandError, match="Invalid format string"):
+            cmd(archiver, "check", "--archives-only", "--format", "{archive")
+
+
+def test_check_format_missing_archive_metadata(archivers, request):
+    # {comment} needs the archive metadata, which is deleted below.
+    archiver = request.getfixturevalue(archivers)
+    check_cmd_setup(archiver)
+    archive, repository = open_archive(archiver.repository_path, "archive1")
+    with repository:
+        repository.delete(archive.id)
+    archive_id_hex = bin_to_hex(archive.id)
+    output = cmd(archiver, "check", "-v", "--archives-only", "--format", "{archive} {comment}", exit_code=1)
+    # the archive directory entry has no name for it, only the id, which {archive} {comment} would not show.
+    # the timestamp uses the same style as the formatter would produce, e.g. "Thu, 1970-01-01 00:00:00 +0000":
+    assert re.search(r"Analyzing archive archive-does-not-exist \w{3}, \d{4}-\d{2}-\d{2} ", output)
+    assert f"{archive_id_hex} (1/2)" in output
+    assert f"Archive metadata block {archive_id_hex} is missing!" in output
+    assert "Analyzing archive archive2" in output  # the intact archive still uses the given format
+
+
 def test_missing_manifest(archivers, request):
     archiver = request.getfixturevalue(archivers)
     check_cmd_setup(archiver)