Skip to content

Commit 76af452

Browse files
payalchapayalcha
authored
Merge branch 'develop' into test-workflow-tls
2 parents e9b5607 + 89f0c2f commit 76af452

File tree

213 files changed

+1691
-7828
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

213 files changed

+1691
-7828
lines changed

‎.github/scripts/nightly_version.sh‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
set -e
3+
4+
if [ "${TEST_PYPI}" = true ]; then
5+
echo "Using Test PyPI"
6+
old_version=$(curl -s https://test.pypi.org/pypi/openfl-nightly/json | python -c "import sys, json; print(json.load(sys.stdin)['info']['version']);")
7+
else
8+
old_version=$(curl -s https://pypi.org/pypi/openfl-nightly/json | python -c "import sys, json; print(json.load(sys.stdin)['info']['version']);")
9+
fi
10+
11+
echo "Old version: $old_version"
12+
13+
version=$(grep -oP "(?<=version=')[^']+" setup.py)
14+
date_suffix=$(date +%Y%m%d)
15+
new_version="${version}${date_suffix}"
16+
17+
# Remove the last digit of old_version after the last character
18+
truncated_old_version=$(echo "${old_version}" | sed 's/.$//')
19+
echo "Truncated old version: $truncated_old_version"
20+
echo "New version: $new_version"
21+
if [ "${truncated_old_version}" == "${new_version}" ]; then
22+
# Increment the last digit of old_version
23+
last_digit=$(echo "${old_version}" | grep -o '.$')
24+
incremented_last_digit=$((last_digit + 1))
25+
new_version="${new_version}${incremented_last_digit}"
26+
else
27+
# Append 0 as the last digit
28+
echo "No version present for current date"
29+
new_version="${new_version}0"
30+
fi
31+
32+
echo "Final NEW_VERSION=${new_version}"
33+
34+
# get URL with commit hash
35+
base_url="https://github.com/securefederatedai/openfl/tree/"
36+
full_url="${base_url}${COMMIT_ID}"
37+
echo "Repository URL: $full_url"
38+
39+
sed -i 's/name=.*/name="openfl-nightly",/' setup.py
40+
sed -i "s/version=.*/version='$new_version',/" setup.py
41+
sed -i 's/Development Status :: 5 - Production\/Stable/Development Status :: 4 - Beta/' setup.py
42+
sed -i "s|'Source Code': '.*'|'Source Code': '${full_url}'|g" setup.py
43+
44+
echo "NEW_VERSION=${new_version}" >> $GITHUB_ENV

‎.github/workflows/experimental_workflow_tests.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,3 @@ jobs:
8383
- name: Workflow - Collaborator Subset (Ray Backend)
8484
run: |
8585
python tests/github/experimental/workflow/LocalRuntime/testflow_subset_of_collaborators.py ray
86-
- name: Test Experimental Aggregator Based Workflow API
87-
run: |
88-
python -m tests.github.experimental.workflow.AggregatorBasedWorkflow.test_experimental_agg_based_workflow --custom_template tests/github/experimental/workflow/AggregatorBasedWorkflow/testcase_datastore_cli --fed_workspace aggregator --col col1 --col col2 --rounds-to-train 1

‎.github/workflows/gandlf.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,6 @@ jobs:
3737
- name: Install dependencies
3838
run: |
3939
python -m pip install --upgrade pip
40-
pip install typer==0.11.1
41-
pip install torch==2.3.1+cpu torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
4240
pip install paramiko
4341
pip install .
4442
pip install -r test-requirements.txt
@@ -50,7 +48,6 @@ jobs:
5048
- name: Install GaNDLF repository
5149
uses: actions/checkout@v4
5250
with:
53-
ref: 0.1.1 # Instead of using the latest tag, we are pinning gandlf to an older version as changes are required in torch to work with a later version.
5451
repository: 'mlcommons/GaNDLF'
5552
fetch-depth: 2 # needed for detecting changes
5653
submodules: 'recursive'

‎.github/workflows/publish_nightly_package.yml‎

Lines changed: 61 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,15 @@ on:
1818
required: false
1919
type: string
2020
description: "Commit ID to use for the nightly build"
21+
test_pypi:
22+
required: false
23+
type: boolean
24+
description: "Use Test PyPI instead of PyPI"
2125

2226
env:
2327
COMMIT_ID: ${{ inputs.commit_id || github.sha }} # use commit_id from the calling workflow
28+
TEST_PYPI: ${{ inputs.test_pypi || false }} # use test_pypi from the calling workflow
29+
PYTHON_VERSION: "3.10"
2430

2531
permissions:
2632
id-token: write
@@ -39,15 +45,15 @@ jobs:
3945
with:
4046
ref: ${{ env.COMMIT_ID }}
4147

42-
- name: Modify setup.py for nightly build
48+
- name: Set up Python
49+
id: setup_python
50+
uses: actions/setup-python@v5
51+
with:
52+
python-version: ${{ env.PYTHON_VERSION }}
53+
54+
- name: Set version and other Metadata in setup.py
4355
run: |
44-
echo "Creating package..."
45-
version=$(grep -oP "(?<=version=')[^']+" setup.py)
46-
date_suffix=$(date +%Y%m%d)
47-
new_version="${version}${date_suffix}"
48-
sed -i 's/name=.*/name="openfl-nightly",/' setup.py
49-
sed -i "s/version=.*/version='$new_version',/" setup.py
50-
sed -i 's/Development Status :: 5 - Production\/Stable/Development Status :: 4 - Beta/' setup.py
56+
bash .github/scripts/nightly_version.sh
5157
5258
- name: Build package
5359
id: build_package
@@ -58,27 +64,62 @@ jobs:
5864
echo "OPENFL_NIGHTLY_WHL_FILE=$(ls dist/*.whl)" >> $GITHUB_ENV
5965
echo "Package built successfully!"
6066
61-
- name: Install built package
62-
id: install_package_using_wheel
67+
- name: Verify package installation before pushing to PyPI
6368
run: |
69+
echo "OpenFL version is $(pip list | grep openfl)"
6470
python3 -m pip uninstall -y openfl openfl-nightly
6571
python3 -m pip install ${{ env.OPENFL_NIGHTLY_WHL_FILE }}
6672
python3 -m pip install -r test-requirements.txt
67-
echo "Installed the built wheel file successfully!"
68-
69-
- name: Verify package installation with E2E test
70-
id: verify_package_installation
71-
run: |
72-
echo "OpenFL version is $(pip list | grep openfl)"
73-
python -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \
73+
python3 -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \
7474
-m task_runner_basic --model_name "keras/mnist" \
75-
--num_rounds 1 --num_collaborators 2
75+
--num_rounds 1 --num_collaborators 1
7676
echo "Verified the package installation successfully!"
7777
7878
- name: Upload package to PyPI
7979
run: |
80-
python3 -m twine upload dist/openfl_nightly* --verbose -u __token__ -p ${{ secrets.PYPI_API_TOKEN }}
81-
echo "Package published successfully!"
80+
if [ "${{ env.TEST_PYPI }}" == "true" ]; then
81+
python3 -m twine upload dist/openfl_nightly* --verbose -u __token__ -p ${{ secrets.TEST_PYPI_API_TOKEN }} --repository-url https://test.pypi.org/legacy/
82+
echo "Package published to Test PyPI successfully!"
83+
else
84+
python3 -m twine upload dist/openfl_nightly* --verbose -u __token__ -p ${{ secrets.PYPI_API_TOKEN }}
85+
echo "Package published to PyPI successfully!"
86+
fi
87+
88+
- name: Install built package
89+
id: install_package_using_wheel
90+
run: |
91+
python3 -m pip uninstall -y openfl openfl-nightly
92+
# Install the package with retry logic
93+
MAX_ATTEMPTS=5
94+
WAIT_TIME_IN_SEC=20
95+
if [ "${{ env.TEST_PYPI }}" == "true" ]; then
96+
PIP_CMD="python3 -m pip install -i https://test.pypi.org/simple/ openfl-nightly==${{ env.NEW_VERSION }}"
97+
else
98+
PIP_CMD="python3 -m pip install openfl-nightly==${{ env.NEW_VERSION }}"
99+
fi
100+
for i in $(seq 1 $MAX_ATTEMPTS); do
101+
if $PIP_CMD; then
102+
echo "Successfully installed openfl-nightly using command '$PIP_CMD'"
103+
break
104+
else
105+
echo "Attempt $i/$MAX_ATTEMPTS failed. Retrying in $WAIT_TIME_IN_SEC seconds..."
106+
sleep $WAIT_TIME_IN_SEC
107+
fi
108+
done
109+
# Check if the package was installed successfully
110+
if ! python3 -m pip show openfl-nightly > /dev/null; then
111+
echo "Failed to install openfl-nightly after $MAX_ATTEMPTS attempts."
112+
exit 1
113+
fi
114+
python3 -m pip install -r test-requirements.txt
115+
116+
- name: Verify package installation after pushing to PyPI
117+
run: |
118+
echo "OpenFL version is $(pip list | grep openfl)"
119+
python3 -m pytest -s tests/end_to_end/test_suites/task_runner_tests.py \
120+
-m task_runner_basic --model_name "keras/mnist" \
121+
--num_rounds 1 --num_collaborators 1
122+
echo "Verified the package installation successfully!"
82123
83124
- name: Print package information
84125
run: |

‎linters-requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
pre-commit
2-
ruff==0.11.2
2+
ruff==0.11.8

‎openfl-tutorials/README.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ fx experimental activate
3030
- [Federated FedProx PyTorch MNIST Workflow Tutorial](https://github.com/securefederatedai/openfl/tree/develop/openfl-tutorials/experimental/workflow/403_Federated_FedProx_PyTorch_MNIST_Workflow_Tutorial.ipynb): Implements FedProx algorithm for distributed training on MNIST using PyTorch and Workflow API.
3131
- [Keras MNIST with FedProx](https://github.com/securefederatedai/openfl/tree/develop/openfl-tutorials/experimental/workflow/404_Keras_MNIST_with_FedProx.ipynb): Trains a TensorFlow Keras model on MNIST using OpenFL Workflow Interface with FedProx optimization.
3232
- [Federated Evaluation MNIST Workflow Tutorial](https://github.com/securefederatedai/openfl/tree/develop/openfl-tutorials/experimental/workflow/405_MNIST_FederatedEvaluation.ipynb): Shows how to implement Federated Evaluation using Workflow API.
33-
- [Workspace Creation from JupyterNotebook](https://github.com/securefederatedai/openfl/tree/develop/openfl-tutorials/experimental/workflow/1001_Workspace_Creation_from_JupyterNotebook.ipynb): Demonstrates converting a Jupyter Notebook Federated Learning experiment into an OpenFL workspace.
3433

3534
### Advanced
3635

0 commit comments

Comments
 (0)