Skip to content

Commit cf0b4ff

Browse files
payalchapayalcha
authored
Add TaskRunner Analytics E2E workflow and integrate into PQ pipeline (#1638)
* Add TaskRunner Analytics E2E workflow and integrate into PQ pipeline Signed-off-by: Chaurasiya, Payal <[email protected]> * review comments Signed-off-by: Chaurasiya, Payal <[email protected]> * fix rest and add models Signed-off-by: Chaurasiya, Payal <[email protected]> * add rest test Signed-off-by: Chaurasiya, Payal <[email protected]> * add rest test Signed-off-by: Chaurasiya, Payal <[email protected]> * comment restapi Signed-off-by: Chaurasiya, Payal <[email protected]> * Add Smokers health Signed-off-by: Chaurasiya, Payal <[email protected]> * remove logs Signed-off-by: Chaurasiya, Payal <[email protected]> * Add review comments Signed-off-by: Chaurasiya, Payal <[email protected]> * Review comments Signed-off-by: Chaurasiya, Payal <[email protected]> * restructure pq Signed-off-by: Chaurasiya, Payal <[email protected]> --------- Signed-off-by: Chaurasiya, Payal <[email protected]>
1 parent e9f0918 commit cf0b4ff

File tree

6 files changed

+196
-3
lines changed

6 files changed

+196
-3
lines changed

‎.github/workflows/pq_pipeline.yml‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ jobs:
8080
(github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
8181
(github.event_name == 'workflow_dispatch')
8282
name: TaskRunner E2E
83-
needs: set_commit_id_for_all_jobs
83+
needs: task_runner_connectivity_e2e
8484
uses: ./.github/workflows/task_runner_basic_e2e.yml
8585
with:
8686
commit_id: ${{ needs.set_commit_id_for_all_jobs.outputs.commit_id }}
@@ -90,7 +90,7 @@ jobs:
9090
(github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
9191
(github.event_name == 'workflow_dispatch')
9292
name: TaskRunner Resiliency E2E
93-
needs: task_runner_e2e
93+
needs: task_runner_connectivity_e2e
9494
uses: ./.github/workflows/task_runner_resiliency_e2e.yml
9595
with:
9696
commit_id: ${{ needs.set_commit_id_for_all_jobs.outputs.commit_id }}
@@ -158,6 +158,16 @@ jobs:
158158
with:
159159
commit_id: ${{ needs.set_commit_id_for_all_jobs.outputs.commit_id }}
160160

161+
task_runner_fed_analytics_e2e:
162+
if: |
163+
(github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
164+
(github.event_name == 'workflow_dispatch')
165+
name: TaskRunner Federated Analytics E2E
166+
needs: task_runner_connectivity_e2e
167+
uses: ./.github/workflows/task_runner_fed_analytics_e2e.yml
168+
with:
169+
commit_id: ${{ needs.set_commit_id_for_all_jobs.outputs.commit_id }}
170+
161171
tr_verifiable_dataset_e2e:
162172
if: |
163173
(github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
@@ -210,6 +220,7 @@ jobs:
210220
wf_secagg_e2e,
211221
task_runner_connectivity_e2e,
212222
task_runner_e2e,
223+
task_runner_fed_analytics_e2e,
213224
task_runner_resiliency_e2e,
214225
task_runner_fedeval_e2e,
215226
task_runner_secure_agg_e2e,

‎.github/workflows/task_runner_fed_analytics_e2e.yml‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
---
2+
# Task Runner Federated Analytics E2E tests for bare metal approach
3+
4+
name: Task_Runner_Fed_Analytics_E2E # Please do not modify the name as it is used in the composite action
5+
6+
on:
7+
workflow_call:
8+
inputs:
9+
commit_id:
10+
required: false
11+
type: string
12+
workflow_dispatch:
13+
inputs:
14+
num_collaborators:
15+
description: "Number of collaborators"
16+
required: false
17+
default: "2"
18+
type: string
19+
python_version:
20+
description: "Python version"
21+
required: false
22+
default: "3.10"
23+
type: choice
24+
options:
25+
- "3.10"
26+
- "3.11"
27+
- "3.12"
28+
29+
permissions:
30+
contents: read
31+
32+
# Environment variables common for all the jobs
33+
# DO NOT use double quotes for the values of the environment variables
34+
env:
35+
NUM_COLLABORATORS: ${{ inputs.num_collaborators || 2 }}
36+
COMMIT_ID: ${{ inputs.commit_id || github.sha }} # use commit_id from the calling workflow
37+
38+
jobs:
39+
test_fed_analytics_histogram:
40+
name: With TLS (federated_analytics/histogram, 3.11) # DO NOT change this name.
41+
runs-on: ubuntu-22.04
42+
timeout-minutes: 30
43+
if: |
44+
(github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
45+
(github.event_name == 'workflow_dispatch') ||
46+
(github.event.pull_request.draft == false)
47+
env:
48+
MODEL_NAME: 'federated_analytics/histogram'
49+
PYTHON_VERSION: ${{ inputs.python_version || '3.11' }}
50+
51+
steps:
52+
- name: Checkout OpenFL repository
53+
id: checkout_openfl
54+
uses: actions/checkout@v4
55+
with:
56+
ref: ${{ env.COMMIT_ID }}
57+
58+
- name: Pre test run
59+
uses: ./.github/actions/tr_pre_test_run
60+
if: ${{ always() }}
61+
62+
- name: Run Federated Analytics Histogram
63+
id: run_tests
64+
run: |
65+
python -m pytest -s tests/end_to_end/test_suites/tr_fed_analytics_tests.py \
66+
-m task_runner_fed_analytics --model_name ${{ env.MODEL_NAME }} --num_collaborators ${{ env.NUM_COLLABORATORS }}
67+
echo "Federated analytics histogram test run completed"
68+
69+
- name: Post test run
70+
uses: ./.github/actions/tr_post_test_run
71+
if: ${{ always() }}
72+
with:
73+
test_type: "Sepal_Histogram_Analytics"
74+
75+
test_fed_analytics_smokers_health:
76+
name: With TLS (federated_analytics/smokers_health, 3.12) # DO NOT change this name.
77+
runs-on: ubuntu-22.04
78+
timeout-minutes: 30
79+
if: |
80+
(github.event_name == 'schedule' && github.repository_owner == 'securefederatedai') ||
81+
(github.event_name == 'workflow_dispatch') ||
82+
(github.event.pull_request.draft == false)
83+
env:
84+
MODEL_NAME: 'federated_analytics/smokers_health'
85+
PYTHON_VERSION: ${{ inputs.python_version || '3.12' }}
86+
steps:
87+
- name: Checkout OpenFL repository
88+
id: checkout_openfl
89+
uses: actions/checkout@v4
90+
with:
91+
ref: ${{ env.COMMIT_ID }}
92+
93+
- name: Pre test run
94+
uses: ./.github/actions/tr_pre_test_run
95+
if: ${{ always() }}
96+
97+
- name: Run Federated Analytics Smokers Health
98+
id: run_tests
99+
run: |
100+
python -m pytest -s tests/end_to_end/test_suites/tr_fed_analytics_tests.py \
101+
-m task_runner_fed_analytics --model_name ${{ env.MODEL_NAME }} --num_collaborators ${{ env.NUM_COLLABORATORS }}
102+
echo "Federated analytics smokers health test run completed"
103+
104+
- name: Post test run
105+
uses: ./.github/actions/tr_post_test_run
106+
if: ${{ always() }}
107+
with:
108+
test_type: "Smokers_Health_Analytics"

‎tests/end_to_end/models/model_owner.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,7 @@ def modify_plan(self, param_config, plan_path):
164164
data["network"]["settings"]["require_client_auth"] = param_config.require_client_auth
165165
data["network"]["settings"]["use_tls"] = param_config.use_tls
166166
if param_config.tr_rest_api:
167-
data["task_runner"]["settings"]["transport_protocol"] = "rest"
167+
data["network"]["settings"]["transport_protocol"] = "rest"
168168
if param_config.secure_agg:
169169
data["aggregator"]["settings"]["secure_aggregation"] = True
170170
with open(plan_file, "w+") as write_file:

‎tests/end_to_end/pytest.ini‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,6 @@ markers =
1313
task_runner_with_s3: mark a test as a task runner with S3 test.
1414
federated_runtime_301_watermarking: mark a test as a federated runtime 301 watermarking test.
1515
straggler_tests: mark a test as a straggler test.
16+
task_runner_fed_analytics: mark a test as a task runner analytics test.
1617
asyncio_mode=auto
1718
asyncio_default_fixture_loop_scope="function"

‎tests/end_to_end/test_suites/tr_fed_analytics_tests.py‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
# Copyright 2020-2025 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
import pytest
5+
import logging
6+
import os
7+
8+
9+
from tests.end_to_end.utils.tr_common_fixtures import (
10+
fx_federation_tr,
11+
)
12+
from tests.end_to_end.utils import federation_helper as fed_helper
13+
import json
14+
import tests.end_to_end.utils.constants as constants
15+
16+
log = logging.getLogger(__name__)
17+
18+
# write a fixture to update request.config.num_rounds to 1
19+
@pytest.fixture(scope="function")
20+
def set_num_rounds(request):
21+
"""
22+
Fixture to set the number of rounds for the test.
23+
Args:
24+
request (Fixture): Pytest fixture
25+
"""
26+
# Set the number of rounds to 1
27+
log.info("Setting number of rounds to 1 for analytics test")
28+
request.config.num_rounds = 1
29+
if "federated_analytics" not in request.config.model_name:
30+
pytest.skip(
31+
f"Model name {request.config.model_name} is not supported for this test. "
32+
"Please use a different model name."
33+
)
34+
35+
36+
@pytest.mark.task_runner_fed_analytics
37+
def test_federation_analytics(request, set_num_rounds, fx_federation_tr):
38+
"""
39+
Test federation via native task runner.
40+
Args:
41+
request (Fixture): Pytest fixture
42+
fx_federation_tr (Fixture): Pytest fixture for native task runner
43+
"""
44+
# Start the federation
45+
assert fed_helper.run_federation(fx_federation_tr)
46+
47+
# Verify the completion of the federation run
48+
assert fed_helper.verify_federation_run_completion(
49+
fx_federation_tr,
50+
test_env=request.config.test_env,
51+
num_rounds=request.config.num_rounds,
52+
), "Federation completion failed"
53+
54+
# verify that results get saved in save/results.json
55+
result_path = os.path.join(
56+
fx_federation_tr.aggregator.workspace_path,
57+
"save",
58+
"result.json"
59+
)
60+
assert os.path.exists(result_path), f"Results file {result_path} does not exist"
61+
62+
with open(result_path, "r") as f:
63+
results = f.read()
64+
try:
65+
results_json = json.loads(results)
66+
except json.JSONDecodeError as e:
67+
log.warning("Results file is not valid JSON. Raw content:\n%s", results)
68+
raise e
69+
70+
assert results, f"Results file {result_path} is empty"

‎tests/end_to_end/utils/constants.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ class ModelName(Enum):
2222
GANDLF_SEG_TEST = "gandlf_seg_test"
2323
FLOWER_APP_PYTORCH = "flower-app-pytorch"
2424
NO_OP = "no-op"
25+
KERAS_TENSORFLOW_MNIST = "keras/tensorflow/mnist"
26+
FEDERATED_ANALYTICS_HISTOGRAM = "federated_analytics/histogram"
27+
FEDERATED_ANALYTICS_SMOKERS_HEALTH = "federated_analytics/smokers_health"
2528

2629
NUM_COLLABORATORS = 2
2730
NUM_ROUNDS = 5

0 commit comments

Comments
 (0)