PBM use pytest shard for parallelization

.github/workflows/PBM-FULL.yml

@@ -25,8 +25,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        psmdb: ["6.0", "7.0", "8.0"]
-        test: [logical, physical, incremental, external]
+        psmdb: ["7.0", "8.0"]
+        shard: [0, 1, 2, 3, 4, 5, 6]
     env:
       PBM_BRANCH: ${{ github.event.inputs.pbm_branch || 'dev' }}
       GO_VER: ${{ github.event.inputs.go_ver || 'bookworm' }}
@@ -38,9 +38,9 @@ jobs:
         PSMDB=perconalab/percona-server-mongodb:${{ matrix.psmdb }} docker compose build
         docker compose up -d
       working-directory: ./pbm-functional/pytest
-    - name: Test ${{ matrix.test }} backup/restore on PSMDB ${{ matrix.psmdb }} and PBM branch ${{ env.PBM_BRANCH }}
+    - name: Run pytest shard ${{ matrix.shard }} on PSMDB ${{ matrix.psmdb }} and PBM branch ${{ env.PBM_BRANCH }}
       run: |
-        docker compose run --rm test pytest -s --junitxml=junit.xml -k ${{ matrix.test }}
+        docker compose run --rm test pytest -s --junitxml=junit.xml --shard-id=${{ matrix.shard }} --num-shards=7 -m 'not jenkins and not skip'
       working-directory: ./pbm-functional/pytest
     - name: Publish Test Report
       uses: mikepenz/action-junit-report@v5

pbm-functional/pytest/Dockerfile-testinfra

@@ -3,6 +3,7 @@ RUN apk add --no-cache docker libffi-dev build-base bash git
 RUN pip install --no-cache-dir --break-system-packages docker && \
     pip install --no-cache-dir --break-system-packages pytest-testinfra && \
     pip install --no-cache-dir --break-system-packages pytest-timeout && \
+    pip install --no-cache-dir --break-system-packages pytest-shard && \
     pip install --no-cache-dir --break-system-packages pymongo pyyaml && \
-    pip install --no-cache-dir --break-system-packages azure-storage-blob
+    pip install --no-cache-dir --break-system-packages azure-storage-blob 
 WORKDIR /test

pbm-functional/pytest/pyproject.toml

@@ -1,3 +1,3 @@
 [tool.pytest.ini_options]
 addopts = "-p no:cacheprovider"
-markers = ["jenkins: mark test to run only in Jenkins or with --jenkins flag"]
+markers = ["jenkins: mark test to run only in Jenkins or with --jenkins flag"]

pbm-functional/pytest/test_PBM-1058_upgrade.py

@@ -43,8 +43,9 @@ def start_cluster(cluster,request):
             cluster.get_logs()
         cluster.destroy(cleanup_backups=True)
 
+@pytest.mark.skip(reason="Outdated")
 @pytest.mark.timeout(300,func_only=True)
-def test_disabled_PBM_T235(start_cluster,cluster):
+def test_physical_PBM_T235(start_cluster,cluster):
     cluster.check_pbm_status()
     pymongo.MongoClient(cluster.connection)["test"]["test"].insert_many(documents)
     backup=cluster.make_backup("physical")

pbm-functional/pytest/test_PBM-1090.py

@@ -31,6 +31,7 @@ def start_cluster(cluster,request):
             cluster.get_logs()
         cluster.destroy(cleanup_backups=True)
 
+@pytest.mark.jenkins
 @pytest.mark.timeout(300,func_only=True)
 def test_load_PBM_T204(start_cluster,cluster):
     client = pymongo.MongoClient(cluster.connection)

pbm-functional/pytest/test_PBM-1223.py

@@ -78,6 +78,7 @@ def test_logical_PBM_T249(start_cluster,cluster):
     assert pymongo.MongoClient(cluster.connection)["test"].command("collstats", "test").get("sharded", False)
     Cluster.log("Finished successfully\n")
 
+@pytest.mark.jenkins
 @pytest.mark.timeout(3600,func_only=True)
 @pytest.mark.parametrize('backup_t',['logic','phys'])
 def test_load_PBM_T272(start_cluster,cluster,backup_t):

pbm-functional/pytest/test_PBM-1252.py

@@ -29,6 +29,7 @@ def start_cluster(cluster,request):
             cluster.get_logs()
         cluster.destroy(cleanup_backups=True)
 
+@pytest.mark.jenkins
 @pytest.mark.timeout(3600,func_only=True)
 def test_load_PBM_T250(start_cluster,cluster):
     cluster.check_pbm_status()

pbm-functional/pytest/test_PBM-1265.py

@@ -45,8 +45,9 @@ def start_cluster(cluster,request):
         cluster.destroy()
         os.system("rm -rf /backups/*")
 
+@pytest.mark.skip(reason="Will be fixed in PBM-1265")
 @pytest.mark.timeout(1200, func_only=True)
-def test_disabled_pitr_PBM_T251(start_cluster,cluster):
+def test_logical_pitr_PBM_T251(start_cluster,cluster):
     cluster.check_pbm_status()
     for i in range(30):
          pymongo.MongoClient(cluster.connection)["test"]["test"].insert_one({"a": i})

pbm-functional/pytest/test_PBM-1355.py

@@ -132,12 +132,14 @@ def test_incremental_PBM_T258(start_cluster,cluster):
     assert pymongo.MongoClient(cluster.connection)["test"]["test"].count_documents({}) == 600
     assert pymongo.MongoClient(cluster.connection)["test"].command("collstats", "test").get("sharded", False)
 
+@pytest.mark.skip(reason="Outdated")
 @pytest.mark.parametrize('command',['config --force-resync','backup'])
-def test_disabled_cli_PBM_T260(start_cluster,cluster,command):
+def test_cli_PBM_T260(start_cluster,cluster,command):
     result = cluster.exec_pbm_cli(command + ' --wait')
     assert result.rc == 0, result.stderr
     Cluster.log(result.stdout)
 
+@pytest.mark.jenkins
 @pytest.mark.timeout(900,func_only=True)
 @pytest.mark.parametrize('restore_ns',['sharded','unsharded'])
 @pytest.mark.parametrize('restore_type',['base','pitr'])

pbm-functional/pytest/test_PBM-1497.py

@@ -49,6 +49,7 @@ def insert_docs(connection,duration):
         if time.time() > timeout:
             break
 
+@pytest.mark.jenkins
 @pytest.mark.timeout(600,func_only=True)
 def test_load_shard_collection_pitr_PBM_T289(start_cluster,cluster):
     cluster.make_backup('logical')

pbm-functional/pytest/test_PBM-1507.py

@@ -1,5 +1,3 @@
-import os
-
 import docker
 import pymongo
 import pytest
@@ -19,97 +17,88 @@ def docker_client():
 def pbm_mongodb_uri():
     return 'mongodb://pbm:[email protected]:27017/?authSource=admin&serverSelectionTimeoutMS=10000'
 
-@pytest.fixture
-def rs_encrypted_same_key(pbm_mongodb_uri):
-    config = {
-        "_id": "rs1",
-        "members": [{"host": "rs101"}, {"host": "rs102"}, {"host": "rs103"}]}
-    return Cluster(config, pbm_mongodb_uri=pbm_mongodb_uri, mongod_extra_args="--enableEncryption --encryptionKeyFile=/etc/mongodb-keyfile")
-
 @pytest.fixture
 def rs_encrypted_mixed_key(pbm_mongodb_uri):
     config = {
         "_id": "rs1",
         "members": [
-            {"host": "rs101"},{"host": "rs102"},
+            {"host": "rs101", "mongod_extra_args": "--enableEncryption --encryptionKeyFile=/etc/mongodb-keyfile"},
+            {"host": "rs102", "mongod_extra_args": "--enableEncryption --encryptionKeyFile=/etc/mongodb-keyfile-new"},
             {"host": "rs103", "mongod_extra_args": "--enableEncryption --encryptionKeyFile=/etc/mongodb-keyfile-new"}]}
-    return Cluster(config, pbm_mongodb_uri=pbm_mongodb_uri, mongod_extra_args="--enableEncryption --encryptionKeyFile=/etc/mongodb-keyfile")
+    return Cluster(config, pbm_mongodb_uri=pbm_mongodb_uri)
 
 @pytest.fixture
 def rs_encrypted_new_key(pbm_mongodb_uri):
     config = {
         "_id": "rs1",
         "members": [{"host": "rs101"}, {"host": "rs102"}, {"host": "rs103"}]}
-    return Cluster(config, pbm_mongodb_uri=pbm_mongodb_uri, mongod_extra_args="--enableEncryption --encryptionKeyFile=/etc/mongodb-keyfile-new")
+    return Cluster(config, pbm_mongodb_uri=pbm_mongodb_uri, mongod_extra_args="--enableEncryption --encryptionKeyFile=/etc/mongodb-keyfile")
 
 @pytest.fixture
-def start_cluster(request):
-    cluster_name, allow_partly_done, cleanup_backups = request.param
-    cluster = request.getfixturevalue(cluster_name)
+def start_cluster(rs_encrypted_mixed_key, request):
+    cluster = rs_encrypted_mixed_key
     try:
         cluster.destroy()
         cluster.create()
-        os.chmod("/backups", 0o777)
-        os.system("rm -rf /backups/*")
         cluster.setup_pbm()
-        yield cluster, allow_partly_done, cluster_name
+        yield cluster
     finally:
         if request.config.getoption("--verbose"):
             cluster.get_logs()
-        cluster.destroy(cleanup_backups=cleanup_backups)
+        cluster.destroy(cleanup_backups=True)
 
 @pytest.mark.timeout(800, func_only=True)
-# RS type, allow_partly_done, cleanup_backups
-@pytest.mark.parametrize("start_cluster",
-        [("rs_encrypted_same_key", False, False),
-        ("rs_encrypted_mixed_key", True, False),
-        ("rs_encrypted_mixed_key", False, False),
-        ("rs_encrypted_new_key", False, True)],
-        ids=["same_key","mixed_key","mixed_key_fb","new_key_fb"],indirect=True)
-@pytest.mark.parametrize('backup_type',['physical','incremental'])
-def test_general_PBM_T299(start_cluster, docker_client, backup_type):
+@pytest.mark.parametrize(
+    "allow_partly_done, start_new_cluster",
+    [(True, False), (False, False), (False, True)],
+    ids=["mixed_key_no_fb","mixed_key_fb","new_key_fb"])
+@pytest.mark.parametrize('backup_type', ['physical', 'incremental'])
+def test_general_PBM_T299(start_cluster, docker_client, allow_partly_done, start_new_cluster, rs_encrypted_new_key, backup_type):
     """
     Test fallback feature functionality during PBM physical restore. Due to configurations
     where database encryption keys differ between backup and running DB, restore may
     partially or fully fail, triggering fallback mechanism.
     Behavior:
-    - Creates backup only during first test run
-    - For 'rs_encrypted_mixed_key' (with allow_partly_done=False) and 'rs_encrypted_new_key':
+    - Creates backup
+    - For mixed_key_fb and new_key_fb:
         Pre-restore data is expected to persist due to fallback on restore failure
-    - For 'rs_encrypted_mixed_key' (with allow_partly_done=True) and 'rs_encrypted_same_key':
+    - For mixed_key_no_fb:
         Only backup data should be present after restore (no fallback triggered)
     - Verifies that .fallbacksync directory is cleaned up after restore on all nodes
     """
-    cluster, allow_partly_done, cluster_name = start_cluster
-    fallback_expected = (cluster_name == "rs_encrypted_new_key" or
-                         (cluster_name == "rs_encrypted_mixed_key" and allow_partly_done is False))
+    cluster = start_cluster
+    fallback_expected = (allow_partly_done is False)
     cluster.check_pbm_status()
     collection = pymongo.MongoClient(cluster.connection)["test"]["test"]
-    cache_entry = backup_cache.get(backup_type)
-    if not cache_entry:
-        inserted_docs = []
-        collection.insert_many(documents)
-        inserted_docs.extend(documents)
-        if backup_type == "physical":
-            backup = cluster.make_backup("physical")
-        elif backup_type == "incremental":
-            cluster.make_backup("incremental --base")
-            collection.insert_many(incr_docs)
-            inserted_docs.extend(incr_docs)
-            backup = cluster.make_backup("incremental")
-        backup_cache[backup_type] = {
-            "backup": backup,
-            "inserted_docs": inserted_docs
-        }
-        assert collection.delete_many({}).deleted_count == len(inserted_docs), "Document deletion failed"
+    inserted_docs = []
+    collection.insert_many(documents)
+    inserted_docs.extend(documents)
+    if backup_type == "physical":
+        backup = cluster.make_backup("physical")
+    elif backup_type == "incremental":
+        cluster.make_backup("incremental --base")
+        collection.insert_many(incr_docs)
+        inserted_docs.extend(incr_docs)
+        backup = cluster.make_backup("incremental")
+    assert collection.delete_many({}).deleted_count == len(inserted_docs), "Document deletion failed"
+
+    if start_new_cluster:
+        cluster.destroy()
+        rs_encrypted_new_key.create()
+        rs_encrypted_new_key.setup_pbm()
+        rs_encrypted_new_key.check_pbm_status()
+        pre_restore_docs = [{"_id": 100 + i, "name": f"pre_{i}", "value": i} for i in range(10)]
+        collection.insert_many(pre_restore_docs)
+        rs_encrypted_new_key.make_restore(backup, timeout=500,
+            restart_cluster=True, check_pbm_status=True,
+            restore_opts=["--fallback-enabled=true", "--allow-partly-done=" + str(allow_partly_done).lower()])
     else:
-        backup = cache_entry["backup"]
-        inserted_docs = cache_entry["inserted_docs"]
-        Cluster.log(f"Using existing backup: {backup}")
-    pre_restore_docs = [{"_id": 100 + i, "name": f"pre_{i}", "value": i} for i in range(10)]
-    collection.insert_many(pre_restore_docs)
-    cluster.make_restore(backup, timeout=500, restart_cluster=True, check_pbm_status=True,
+        pre_restore_docs = [{"_id": 100 + i, "name": f"pre_{i}", "value": i} for i in range(10)]
+        collection.insert_many(pre_restore_docs)
+        cluster.make_restore(backup, timeout=500,
+            restart_cluster=True, check_pbm_status=True,
             restore_opts=["--fallback-enabled=true", "--allow-partly-done=" + str(allow_partly_done).lower()])
+
     expected_docs = pre_restore_docs if fallback_expected else inserted_docs
     restored_count = collection.count_documents({})
     assert restored_count == len(expected_docs), f"Expected {len(expected_docs)} documents, found {restored_count}"

pbm-functional/pytest/test_PBM-1531.py

@@ -64,6 +64,7 @@ def start_cluster(cluster,request):
         wrong_rscfg03_cluster.destroy()
         cluster.destroy()
 
+@pytest.mark.skip(reason="Skipped because it is possible STR")
 def test_STR_PBM_T292(start_cluster,cluster):
     coll = "test"
     for j in range(10):

pbm-functional/pytest/test_drop_pitr.py

@@ -35,12 +35,13 @@ def start_cluster(cluster,request):
             cluster.get_logs()
         cluster.destroy(cleanup_backups=True)
 
+@pytest.mark.skip(reason="Not yet supported")
 @pytest.mark.timeout(600,func_only=True)
 @pytest.mark.parametrize('restore_type',['full','selective'])
 @pytest.mark.parametrize('primary_shard',['unchanged','changed'])
 @pytest.mark.parametrize('old_collection',['sharded','unsharded'])
 @pytest.mark.parametrize('new_collection',['sharded','unsharded'])
-def test_disabled_drop_pitr_PBM_T281(start_cluster,cluster,restore_type,primary_shard,old_collection,new_collection):
+def test_drop_pitr_PBM_T281(start_cluster,cluster,restore_type,primary_shard,old_collection,new_collection):
     cluster.check_pbm_status()
     client = pymongo.MongoClient(cluster.connection)
     # the primary shard for old database - rs1

pbm-functional/pytest/test_sharded_chunks.py

@@ -55,6 +55,7 @@ def insert_docs(connection,duration):
         if time.time() > timeout:
             break
 
+@pytest.mark.jenkins
 @pytest.mark.timeout(360,func_only=True)
 def test_load_chunks_migration_pitr_PBM_T286(start_cluster,cluster):
     cluster.make_backup('logical')
@@ -90,7 +91,7 @@ def test_load_chunks_migration_pitr_PBM_T286(start_cluster,cluster):
     assert pymongo.MongoClient(cluster.connection)["test"].command("collstats", "test").get("sharded", False)
     Cluster.log("Finished successfully")
 
-
+@pytest.mark.jenkins
 @pytest.mark.timeout(180,func_only=True)
 def test_load_chunks_migration_base_PBM_T285(start_cluster,cluster):
     threads = 100

pbm-functional/pytest/test_timeseries.py

@@ -46,8 +46,9 @@ def start_cluster(cluster,request):
             cluster.get_logs()
         cluster.destroy(cleanup_backups=True)
 
+@pytest.mark.skip(reason="Not yet supported")
 @pytest.mark.timeout(600,func_only=True)
-def test_disabled_PBM_T252(start_cluster,cluster):
+def test_logical_PBM_T252(start_cluster,cluster):
     cluster.check_pbm_status()
     assert pymongo.MongoClient(cluster.connection)["test"].command("collstats", "test1").get("sharded", False)
     assert pymongo.MongoClient(cluster.connection)["test"].command("collstats", "test2").get("sharded", False)
@@ -62,8 +63,9 @@ def test_disabled_PBM_T252(start_cluster,cluster):
     assert pymongo.MongoClient(cluster.connection)["test"].command("collstats", "test2").get("sharded", False)
     Cluster.log("Finished successfully")
 
+@pytest.mark.skip(reason="Not yet supported")
 @pytest.mark.timeout(600,func_only=True)
-def test_disabled_without_data(start_cluster,cluster):
+def test_logical_without_data(start_cluster,cluster):
     cluster.check_pbm_status()
     assert pymongo.MongoClient(cluster.connection)["test"].command("collstats", "test1").get("sharded", False)
     assert pymongo.MongoClient(cluster.connection)["test"].command("collstats", "test2").get("sharded", False)
@@ -92,7 +94,7 @@ def test_incremental_PBM_T262(start_cluster,cluster):
     for i in range(10):
         pymongo.MongoClient(cluster.connection)["test"]["test1"].insert_one({"timestamp": datetime.now(), "data": i})
         time.sleep(0.1)
-    base_backup=cluster.make_backup("incremental --base")
+    cluster.make_backup("incremental --base")
     for i in range(10):
         pymongo.MongoClient(cluster.connection)["test"]["test2"].insert_one({"timestamp": datetime.now(), "data": i})
         time.sleep(0.1)