Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
154 changes: 33 additions & 121 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -29,107 +29,8 @@ permissions:
contents: read

jobs:
lint:

runs-on: ubuntu-22.04
timeout-minutes: 5

steps:
- uses: actions/checkout@v6
- uses: astral-sh/ruff-action@v3

security:

runs-on: ubuntu-24.04
timeout-minutes: 5

steps:
- uses: actions/checkout@v6
- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.10'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install bandit[toml]
- name: Run Bandit
run: |
bandit -r src/borg -c pyproject.toml

asan_ubsan:

runs-on: ubuntu-24.04
timeout-minutes: 25
needs: [lint]

steps:
- uses: actions/checkout@v6
with:
# Just fetching one commit is not enough for setuptools-scm, so we fetch all.
fetch-depth: 0
fetch-tags: true

- name: Set up Python
uses: actions/setup-python@v6
with:
python-version: '3.12'

- name: Install system packages
run: |
sudo apt-get update
sudo apt-get install -y pkg-config build-essential
sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev

- name: Install Python dependencies
run: |
python -m pip install --upgrade pip
pip install -r requirements.d/development.lock.txt

- name: Build Borg with ASan/UBSan
# Build the C/Cython extensions with AddressSanitizer and UndefinedBehaviorSanitizer enabled.
# How this works:
# - The -fsanitize=address,undefined flags inject runtime checks into our native code. If a bug is hit
# (e.g., buffer overflow, use-after-free, out-of-bounds, or undefined behavior), the sanitizer prints
# a detailed error report to stderr, including a stack trace, and forces the process to exit with
# non-zero status. In CI, this will fail the step/job so you will notice.
# - ASAN_OPTIONS/UBSAN_OPTIONS configure the sanitizers' runtime behavior (see below for meanings).
env:
CFLAGS: "-O1 -g -fno-omit-frame-pointer -fsanitize=address,undefined"
CXXFLAGS: "-O1 -g -fno-omit-frame-pointer -fsanitize=address,undefined"
LDFLAGS: "-fsanitize=address,undefined"
# ASAN_OPTIONS controls AddressSanitizer runtime tweaks:
# - detect_leaks=0: Disable LeakSanitizer to avoid false positives with CPython/pymalloc in short-lived tests.
# - strict_string_checks=1: Make invalid string operations (e.g., over-reads) more likely to be detected.
# - check_initialization_order=1: Catch uses that depend on static initialization order (C++).
# - detect_stack_use_after_return=1: Detect stack-use-after-return via stack poisoning (may increase overhead).
ASAN_OPTIONS: "detect_leaks=0:strict_string_checks=1:check_initialization_order=1:detect_stack_use_after_return=1"
# UBSAN_OPTIONS controls UndefinedBehaviorSanitizer runtime:
# - print_stacktrace=1: Include a stack trace for UB reports to ease debugging.
# Note: UBSan is recoverable by default (process may continue after reporting). If you want CI to
# abort immediately and fail on the first UB, add `halt_on_error=1` (e.g., UBSAN_OPTIONS="print_stacktrace=1:halt_on_error=1").
UBSAN_OPTIONS: "print_stacktrace=1"
# PYTHONDEVMODE enables additional Python runtime checks and warnings.
PYTHONDEVMODE: "1"
run: pip install -e .

- name: Run tests under sanitizers
env:
ASAN_OPTIONS: "detect_leaks=0:strict_string_checks=1:check_initialization_order=1:detect_stack_use_after_return=1"
UBSAN_OPTIONS: "print_stacktrace=1"
PYTHONDEVMODE: "1"
# Ensure the ASan runtime is loaded first to avoid "ASan runtime does not come first" warnings.
# We discover libasan/libubsan paths via gcc and preload them for the Python test process.
# the remote tests are slow and likely won't find anything useful
run: |
set -euo pipefail
export LD_PRELOAD="$(gcc -print-file-name=libasan.so):$(gcc -print-file-name=libubsan.so)"
echo "Using LD_PRELOAD=$LD_PRELOAD"
pytest -v --benchmark-skip -k "not remote"

native_tests:

needs: [lint]
permissions:
contents: read
id-token: write
Expand All @@ -141,21 +42,10 @@ jobs:
${{ fromJSON(
github.event_name == 'pull_request' && '{
"include": [
{"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "mypy"},
{"os": "ubuntu-22.04", "python-version": "3.11", "toxenv": "docs"},
{"os": "ubuntu-22.04", "python-version": "3.10", "toxenv": "py310-llfuse"},
{"os": "ubuntu-24.04", "python-version": "3.12", "toxenv": "py312-pyfuse3"},
{"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-mfusepy"}
{"os": "macos-15", "python-version": "3.11", "toxenv": "py311-none", "binary": "borg-macos-15-arm64-gh"},
]
}' || '{
"include": [
{"os": "ubuntu-22.04", "python-version": "3.11", "toxenv": "py311-pyfuse3", "binary": "borg-linux-glibc235-x86_64-gh"},
{"os": "ubuntu-22.04-arm", "python-version": "3.11", "toxenv": "py311-pyfuse3", "binary": "borg-linux-glibc235-arm64-gh"},
{"os": "ubuntu-24.04", "python-version": "3.12", "toxenv": "py312-llfuse"},
{"os": "ubuntu-24.04", "python-version": "3.13", "toxenv": "py313-pyfuse3"},
{"os": "ubuntu-24.04", "python-version": "3.14", "toxenv": "py314-mfusepy"},
{"os": "macos-15", "python-version": "3.11", "toxenv": "py311-none", "binary": "borg-macos-15-arm64-gh"},
{"os": "macos-15-intel", "python-version": "3.11", "toxenv": "py311-none", "binary": "borg-macos-15-x86_64-gh"}
]
}'
) }}
Expand Down Expand Up @@ -380,19 +270,18 @@ jobs:
attestations: write
runs-on: ubuntu-24.04
timeout-minutes: 180
needs: [lint]
continue-on-error: true

strategy:
fail-fast: false
matrix:
include:
- os: freebsd
version: '14.3'
version: '15.0'
display_name: FreeBSD
# Controls binary build and provenance attestation on tags
do_binaries: true
artifact_prefix: borg-freebsd-14-x86_64-gh
artifact_prefix: borg-freebsd-15-x86_64-gh

- os: netbsd
version: '10.1'
Expand Down Expand Up @@ -523,6 +412,30 @@ jobs:
touch ${TMPDIR}/testfile
lsextattr user ${TMPDIR}/testfile && echo "[xattr] *** xattrs SUPPORTED on ${TMPDIR}! ***"

# NetBSD 10 has a too old OpenSSL, build a fresher one.
VERSION="3.5.6"
echo "--- Building OpenSSL ${VERSION} ---"
PREFIX="/usr/local"
JOBS=$(sysctl -n hw.ncpu)

pushd /tmp
ftp -o "openssl-${VERSION}.tar.gz" "https://www.openssl.org/source/openssl-${VERSION}.tar.gz"
tar xzf "openssl-${VERSION}.tar.gz"
pushd "openssl-${VERSION}"

./Configure --prefix="${PREFIX}" --openssldir="${PREFIX}/etc/ssl" --libdir=lib \
-Wl,-rpath,${PREFIX}/lib shared threads no-tests no-docs
export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
make -j"${JOBS}"
sudo -E make install
popd
rm -rf "/tmp/openssl-${VERSION}" "/tmp/openssl-${VERSION}.tar.gz"
popd

"${PREFIX}/bin/openssl" version
export BORG_OPENSSL_PREFIX="${PREFIX}"
export PKG_CONFIG_PATH="/usr/local/lib/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}"

tox3 -e py311-none
;;

Expand Down Expand Up @@ -558,11 +471,12 @@ jobs:
haiku)
pkgman refresh
pkgman install -y git pkgconfig lz4
pkgman install -y openssl3
pkgman install -y rust_bin
pkgman install -y python3.10
pkgman install -y cffi
pkgman install -y lz4_devel openssl3_devel libffi_devel
pkgman install -y python3.10 lz4_devel
# haiku r1beta5 has OpenSSL 3.0.14, so we manually pull 3.5.6 from current master:
curl -L -O https://eu.hpkg.haiku-os.org/haikuports/master/x86_64/current/packages/openssl3-3.5.6-1-x86_64.hpkg
curl -L -O https://eu.hpkg.haiku-os.org/haikuports/master/x86_64/current/packages/openssl3_devel-3.5.6-1-x86_64.hpkg
pkgman install -y openssl3-3.5.6-1-x86_64.hpkg openssl3_devel-3.5.6-1-x86_64.hpkg

# there is no pkgman package for tox, so we install it into a venv
python3 -m ensurepip --upgrade
Expand All @@ -572,7 +486,6 @@ jobs:

export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
export BORG_LIBLZ4_PREFIX=/system/develop
export BORG_OPENSSL_PREFIX=/system/develop
pip install -r requirements.d/development.lock.txt
pip install -e .

Expand Down Expand Up @@ -622,7 +535,6 @@ jobs:
if: true # can be used to temporarily disable the build
runs-on: windows-latest
timeout-minutes: 90
needs: [lint]

env:
PY_COLORS: 1
Expand All @@ -648,7 +560,7 @@ jobs:

- name: Build python venv
run: |
# building cffi / argon2-cffi in the venv fails, so we try to use the system packages
# building native extensions in the venv fails, so we try to use the system packages
python -m venv --system-site-packages env
. env/bin/activate
# python -m pip install --upgrade pip
Expand Down
86 changes: 0 additions & 86 deletions .github/workflows/codeql-analysis.yml

This file was deleted.

4 changes: 2 additions & 2 deletions docs/installation.rst
Original file line number Diff line number Diff line change
Expand Up @@ -162,10 +162,10 @@ following dependencies first. For the libraries you will also need their
development header files (sometimes in a separate `-dev` or `-devel` package).

* `Python 3`_ >= 3.10.0
* OpenSSL_ >= 1.1.1 (LibreSSL will not work)
* OpenSSL_ >= 3.2.0 (LibreSSL will not work)
* libacl_ (which depends on libattr_)
* liblz4_ >= 1.7.0 (r129)
* libffi (required for argon2-cffi-bindings)

* pkg-config (cli tool) - Borg uses this to discover header and library
locations automatically. Alternatively, you can also point to them via some
environment variables, see setup.py.
Expand Down
2 changes: 1 addition & 1 deletion docs/internals/security.rst
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,7 @@ on widely used libraries providing them:
primitives implemented in libcrypto.
- SHA-256, SHA-512 and BLAKE2b from Python's hashlib_ standard library module are used.
- HMAC and a constant-time comparison from Python's hmac_ standard library module are used.
- argon2 is used via argon2-cffi.
- argon2 is used from OpenSSL (>= 3.2).

.. _Horton principle: https://en.wikipedia.org/wiki/Horton_Principle
.. _length extension: https://en.wikipedia.org/wiki/Length_extension_attack
Expand Down
1 change: 0 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@ dependencies = [
"packaging",
"platformdirs >=3.0.0, <5.0.0; sys_platform == 'darwin'", # for macOS: breaking changes in 3.0.0.
"platformdirs >=2.6.0, <5.0.0; sys_platform != 'darwin'", # for others: 2.6+ works consistently.
"argon2-cffi",
"shtab>=1.8.0",
"backports-zstd; python_version < '3.14'", # for python < 3.14.
"xxhash>=2.0.0",
Expand Down
2 changes: 1 addition & 1 deletion scripts/msys2-install-deps
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash

pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,xxhash,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko}
pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,xxhash,openssl,rclone,python-msgpack,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko}

if [ "$1" = "development" ]; then
pacman -S --needed --noconfirm mingw-w64-ucrt-x86_64-python-{pytest,pytest-benchmark,pytest-cov,pytest-xdist}
Expand Down
13 changes: 11 additions & 2 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@

is_win32 = sys.platform.startswith("win32")
is_openbsd = sys.platform.startswith("openbsd")
is_netbsd = sys.platform.startswith("netbsd")

# Number of threads to use for cythonize, not used on Windows
cpu_threads = multiprocessing.cpu_count() if multiprocessing and multiprocessing.get_start_method() != "spawn" else None
Expand Down Expand Up @@ -139,7 +140,7 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s
)

if is_win32:
crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "libcrypto", "libcrypto", ">=1.1.1", lib_subdir="")
crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "libcrypto", "libcrypto", ">=3.2.0", lib_subdir="")
elif is_openbsd:
# Use OpenSSL (not LibreSSL) because we need AES-OCB via the EVP API. Link
# it statically to avoid conflicting with shared libcrypto from the base
Expand All @@ -150,8 +151,16 @@ def lib_ext_kwargs(pc, prefix_env_var, lib_name, lib_pkg_name, pc_version, lib_s
include_dirs=[os.path.join(openssl_prefix, "include", openssl_name)],
extra_objects=[os.path.join(openssl_prefix, "lib", openssl_name, "libcrypto.a")],
)
elif is_netbsd and os.environ.get("BORG_OPENSSL_PREFIX"):
# Similarly for NetBSD, if we built a custom OpenSSL, link it statically
# to avoid dynamic linker conflicts with the system OpenSSL loaded by Python.
openssl_prefix = os.environ.get("BORG_OPENSSL_PREFIX")
crypto_ext_lib = dict(
include_dirs=[os.path.join(openssl_prefix, "include")],
extra_objects=[os.path.join(openssl_prefix, "lib", "libcrypto.a")],
)
else:
crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "crypto", "libcrypto", ">=1.1.1")
crypto_ext_lib = lib_ext_kwargs(pc, "BORG_OPENSSL_PREFIX", "crypto", "libcrypto", ">=3.2.0")

crypto_ext_kwargs = members_appended(
dict(sources=[crypto_ll_source]), crypto_ext_lib, dict(extra_compile_args=cflags)
Expand Down
Loading
Loading