add blake3, blake2b is legacy

.github/workflows/ci.yml

@@ -537,7 +537,7 @@ jobs:
                 ;;
 
               omnios)
-                sudo pkg install gcc14 git pkg-config python-313 gnu-make gnu-coreutils
+                sudo pkg install gcc14 git pkg-config python-313 gnu-make gnu-coreutils rust
                 sudo ln -sf /usr/bin/python3.13 /usr/bin/python3
                 sudo ln -sf /usr/bin/python3.13-config /usr/bin/python3-config
                 sudo python3 -m ensurepip
@@ -559,21 +559,25 @@ jobs:
                 pkgman refresh
                 pkgman install -y git pkgconfig lz4
                 pkgman install -y openssl3
-                pkgman install -y rust_bin
+                pkgman install -y rust_bin setuptools_rust
                 pkgman install -y python3.10
                 pkgman install -y cffi
                 pkgman install -y lz4_devel openssl3_devel libffi_devel
 
+                cargo --version
+
                 # there is no pkgman package for tox, so we install it into a venv
                 python3 -m ensurepip --upgrade
-                python3 -m pip install --upgrade pip wheel
+                python3 -m pip install --upgrade pip setuptools wheel
                 python3 -m venv .venv
                 . .venv/bin/activate
 
                 export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
                 export BORG_LIBLZ4_PREFIX=/system/develop
                 export BORG_OPENSSL_PREFIX=/system/develop
                 pip install -r requirements.d/development.lock.txt
+                pip install -v maturin
+                pip install -v blake3
                 pip install -e .
 
                 # troubles with either tox or pytest xdist, so we run pytest manually:

docs/usage/transfer.rst

@@ -2,33 +2,69 @@
 
 Examples
 ~~~~~~~~
+
+To keep the following examples short and readable, we export the repository
+locations and passphrases first:
+
+::
+
+    export BORG_REPO=ssh://borg2@borgbackup/./tests/b20
+    export BORG_PASSPHRASE='your-borg2-repo-passphrase'
+    export BORG_OTHER_REPO=ssh://borg2@borgbackup/./tests/b1x
+    export BORG_OTHER_PASSPHRASE='your-borg1-repo-passphrase'
+
 ::
 
-    # 0. Have Borg 2.0 installed on the client AND server; have a b12 repository copy for testing.
+    # borg 1.x repo -> borg 2.0 repo (hmac-sha256 -> hmac-sha256, keeping same chunk ID algorithm)
+
+    # 0. Have Borg 2.0 installed on the client AND server; have a b1x repository copy for testing.
 
     # 1. Create a new "related" repository:
-    # Here, the existing Borg 1.2 repository used repokey-blake2 (and AES-CTR mode),
-    # thus we use repokey-blake2-aes-ocb for the new Borg 2.0 repository.
-    # Staying with the same chunk ID algorithm (BLAKE2) and with the same
-    # key material (via --other-repo <oldrepo>) will make deduplication work
+    # Here, the existing Borg 1.x repository used repokey (and AES-CTR mode),
+    # thus we use repokey-aes-ocb for the new Borg 2.0 repository.
+    # Staying with the same chunk ID algorithm (hmac-sha256) and with the same
+    # key material (via BORG_OTHER_REPO) will make deduplication work
     # between old archives (copied with borg transfer) and future ones.
     # The AEAD cipher does not matter (everything must be re-encrypted and
-    # re-authenticated anyway); you could also choose repokey-blake2-chacha20-poly1305.
-    # In case your old Borg repository did not use BLAKE2, just remove the "-blake2".
-    $ borg --repo       ssh://borg2@borgbackup/./tests/b20 repo-create \
-           --other-repo ssh://borg2@borgbackup/./tests/b12 -e repokey-blake2-aes-ocb
+    # re-authenticated anyway); you could also choose repokey-chacha20-poly1305.
+    $ borg repo-create -e repokey-aes-ocb
+
+    # 2. Check what and how much it would transfer:
+    $ borg transfer --from-borg1 --dry-run
+
+    # 3. Transfer (copy) archives from the old repository into the new repository (takes time and space!):
+    $ borg transfer --from-borg1
+
+    # 4. Check whether we have everything (same as step 2):
+    $ borg transfer --from-borg1 --dry-run
+
+::
+
+    # borg 1.x repo -> borg 2.0 repo (blake2 -> blake3, changing chunk ID algorithm)
+
+    # 0. Have Borg 2.0 installed on the client AND server; have a b1x repository copy for testing.
+
+    # 1. Create a new "related" repository:
+    # Here, the existing Borg 1.x repository used repokey-blake2 (and AES-CTR mode),
+    # thus we use repokey-blake3-aes-ocb for the new Borg 2.0 repository.
+    # We need to change from blake2 to blake3, because blake2 is not supported
+    # for borg2 repos (blake3 is much faster). Because we change how chunk IDs are
+    # computed, we need to re-chunk everything while doing the transfer.
+    # The chunker parameters you provide here should be the same as you will
+    # use for all future Borg 2.0 archives.
+    # The AEAD cipher does not matter (everything must be re-encrypted and
+    # re-authenticated anyway); you could also choose repokey-blake3-chacha20-poly1305.
+    $ borg repo-create -e repokey-blake3-aes-ocb
+    $ export CHUNKER_PARAMS="buzhash64,19,23,21,4095"
 
     # 2. Check what and how much it would transfer:
-    $ borg --repo       ssh://borg2@borgbackup/./tests/b20 transfer --upgrader=From12To20 \
-           --other-repo ssh://borg2@borgbackup/./tests/b12 --dry-run
+    $ borg transfer --from-borg1 --chunker-params=$CHUNKER_PARAMS --dry-run
 
     # 3. Transfer (copy) archives from the old repository into the new repository (takes time and space!):
-    $ borg --repo       ssh://borg2@borgbackup/./tests/b20 transfer --upgrader=From12To20 \
-           --other-repo ssh://borg2@borgbackup/./tests/b12
+    $ borg transfer --from-borg1 --chunker-params=$CHUNKER_PARAMS
 
     # 4. Check whether we have everything (same as step 2):
-    $ borg --repo       ssh://borg2@borgbackup/./tests/b20 transfer --upgrader=From12To20 \
-           --other-repo ssh://borg2@borgbackup/./tests/b12 --dry-run
+    $ borg transfer --from-borg1 --chunker-params=$CHUNKER_PARAMS --dry-run
 
 Keyfile considerations when upgrading from borg 1.x
 ++++++++++++++++++++++++++++++++++++++++++++++++++++

pyproject.toml

@@ -42,6 +42,7 @@ dependencies = [
   "xxhash>=2.0.0",
   "jsonargparse>=4.47.0",
   "PyYAML>=6.0.2",  # we need to register our types with yaml, jsonargparse uses yaml for config files
+  "blake3>=1.0.0",
 ]
 
 [project.optional-dependencies]

scripts/msys2-install-deps

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,xxhash,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko}
+pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,xxhash,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko,rust,python-maturin}
 
 if [ "$1" = "development" ]; then
 	pacman -S --needed --noconfirm mingw-w64-ucrt-x86_64-python-{pytest,pytest-benchmark,pytest-cov,pytest-xdist}

src/borg/archiver/benchmark_cmd.py

@@ -232,16 +232,20 @@ def chunkit(ch):
                 print(f"{spec:<24} {format_file_size(size):<10} {dt:.3f}s")
 
         from ..crypto.low_level import hmac_sha256, blake2b_256
+        import blake3
 
         if not args.json:
             print("Cryptographic hashes / MACs ====================================")
         else:
             result["hashes"] = []
         size = 1000000000
-        for spec, func in [
+        hashes_tests = [
             ("hmac-sha256", lambda: hmac_sha256(key_256, random_10M)),
             ("blake2b-256", lambda: blake2b_256(key_256, random_10M)),
-        ]:
+            ("blake3", lambda: blake3.blake3(random_10M, key=key_256).digest()),
+            ("blake3-mt", lambda: blake3.blake3(random_10M, key=key_256, max_threads=blake3.blake3.AUTO).digest()),
+        ]
+        for spec, func in hashes_tests:
             dt = timeit(func, number=number_default)
             if args.json:
                 result["hashes"].append({"algo": spec, "size": size, "time": dt})

src/borg/archiver/key_cmds.py

@@ -1,8 +1,8 @@
 import os
 
 from ..constants import *  # NOQA
-from ..crypto.key import AESOCBRepoKey, CHPORepoKey, Blake2AESOCBRepoKey, Blake2CHPORepoKey
-from ..crypto.key import AESOCBKeyfileKey, CHPOKeyfileKey, Blake2AESOCBKeyfileKey, Blake2CHPOKeyfileKey
+from ..crypto.key import AESOCBRepoKey, CHPORepoKey, Blake3AESOCBRepoKey, Blake3CHPORepoKey
+from ..crypto.key import AESOCBKeyfileKey, CHPOKeyfileKey, Blake3AESOCBKeyfileKey, Blake3CHPOKeyfileKey
 from ..crypto.keymanager import KeyManager
 from ..helpers import PathSpec, CommandError
 from ..helpers.argparsing import ArgumentParser
@@ -40,10 +40,10 @@ def do_key_change_location(self, args, repository, manifest, cache):
                 key_new = AESOCBKeyfileKey(repository)
             elif isinstance(key, CHPORepoKey):
                 key_new = CHPOKeyfileKey(repository)
-            elif isinstance(key, Blake2AESOCBRepoKey):
-                key_new = Blake2AESOCBKeyfileKey(repository)
-            elif isinstance(key, Blake2CHPORepoKey):
-                key_new = Blake2CHPOKeyfileKey(repository)
+            elif isinstance(key, Blake3AESOCBRepoKey):
+                key_new = Blake3AESOCBKeyfileKey(repository)
+            elif isinstance(key, Blake3CHPORepoKey):
+                key_new = Blake3CHPOKeyfileKey(repository)
             else:
                 print("Change not needed or not supported.")
                 return
@@ -52,10 +52,10 @@ def do_key_change_location(self, args, repository, manifest, cache):
                 key_new = AESOCBRepoKey(repository)
             elif isinstance(key, CHPOKeyfileKey):
                 key_new = CHPORepoKey(repository)
-            elif isinstance(key, Blake2AESOCBKeyfileKey):
-                key_new = Blake2AESOCBRepoKey(repository)
-            elif isinstance(key, Blake2CHPOKeyfileKey):
-                key_new = Blake2CHPORepoKey(repository)
+            elif isinstance(key, Blake3AESOCBKeyfileKey):
+                key_new = Blake3AESOCBRepoKey(repository)
+            elif isinstance(key, Blake3CHPOKeyfileKey):
+                key_new = Blake3CHPORepoKey(repository)
             else:
                 print("Change not needed or not supported.")
                 return

src/borg/archiver/transfer_cmd.py

@@ -134,12 +134,11 @@ def do_transfer(self, args, *, repository, manifest, cache, other_repository=Non
         """archives transfer from other repository, optionally upgrade data format"""
         key = manifest.key
         other_key = other_manifest.key
-        if not uses_same_id_hash(other_key, key):
-            raise Error(
-                "You must keep the same ID hash ([HMAC-]SHA256 or BLAKE2b) or deduplication will break. "
-                "Use a related repository!"
-            )
-        if not uses_same_chunker_secret(other_key, key):
+        using_same_id_hash = uses_same_id_hash(other_key, key)
+        rechunking = args.chunker_params is not None
+        if not using_same_id_hash and not rechunking:
+            raise Error("You must either keep the same ID hash or use --chunker-params.")
+        if not rechunking and not uses_same_chunker_secret(other_key, key):
             raise Error(
                 "You must use the same chunker secret or deduplication will break. " "Use a related repository!"
             )

src/borg/constants.py

@@ -191,10 +191,11 @@ class KeyType:
     AESOCBREPO = 0x11
     CHPOKEYFILE = 0x20
     CHPOREPO = 0x21
-    BLAKE2AESOCBKEYFILE = 0x30
-    BLAKE2AESOCBREPO = 0x31
-    BLAKE2CHPOKEYFILE = 0x40
-    BLAKE2CHPOREPO = 0x41
+    BLAKE3AESOCBKEYFILE = 0x30
+    BLAKE3AESOCBREPO = 0x31
+    BLAKE3CHPOKEYFILE = 0x40
+    BLAKE3CHPOREPO = 0x41
+    BLAKE3AUTHENTICATED = 0x50
 
 
 CACHE_TAG_NAME = "CACHEDIR.TAG"