Skip to content

Commit 2b94932

Browse files
authored
ci: Git artifact check (#12477)
Signed-off-by: Nelesh Singla <[email protected]>
1 parent ab7e607 commit 2b94932

File tree

5 files changed

+195
-18
lines changed

5 files changed

+195
-18
lines changed
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: 'Check Artifact Exists'
2+
description: 'Check if a GitHub artifact exists in the current workflow run'
3+
author: 'KFP Team'
4+
5+
inputs:
6+
artifact_name:
7+
description: 'Name of the artifact to check'
8+
required: true
9+
10+
outputs:
11+
exists:
12+
description: 'Whether the artifact exists (true/false)'
13+
value: ${{ steps.check.outputs.exists }}
14+
15+
runs:
16+
using: 'composite'
17+
steps:
18+
- name: Setup Python
19+
uses: actions/setup-python@v4
20+
with:
21+
python-version: '3.11'
22+
23+
- name: Install dependencies
24+
shell: bash
25+
run: |
26+
python -m pip install --upgrade pip
27+
pip install requests
28+
29+
- name: Check artifact exists
30+
shell: bash
31+
id: check
32+
run: python ${{ github.action_path }}/check_artifact.py
33+
env:
34+
INPUT_ARTIFACT_NAME: ${{ inputs.artifact_name }}
35+
36+
branding:
37+
icon: 'search'
38+
color: 'blue'
Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
#!/usr/bin/env python3
2+
"""
3+
GitHub Action script to check if an artifact exists in the repository for a specific commit.
4+
Uses the GitHub REST API to list artifacts by name and filters by head SHA.
5+
Returns true if the artifact exists, false otherwise.
6+
"""
7+
8+
import os
9+
import sys
10+
import requests
11+
import json
12+
13+
def get_github_token() -> str:
14+
"""Get GitHub token from environment variables."""
15+
token = os.getenv('GITHUB_TOKEN')
16+
if not token:
17+
print("::error::GITHUB_TOKEN environment variable is required")
18+
sys.exit(1)
19+
return token
20+
21+
def get_head_sha() -> str:
22+
"""Get the head SHA from environment variables."""
23+
head_sha = os.getenv('GITHUB_SHA')
24+
if not head_sha:
25+
print("::error::GITHUB_SHA environment variable is required")
26+
sys.exit(1)
27+
return head_sha
28+
29+
def get_repository() -> str:
30+
"""Get the repository name from environment variables."""
31+
repo = os.getenv('GITHUB_REPOSITORY')
32+
if not repo:
33+
print("::error::GITHUB_REPOSITORY environment variable is required")
34+
sys.exit(1)
35+
return repo
36+
37+
def set_output(name: str, value: str) -> None:
38+
"""Set GitHub Actions output."""
39+
github_output = os.getenv('GITHUB_OUTPUT')
40+
if github_output:
41+
with open(github_output, 'a') as f:
42+
f.write(f"{name}={value}\n")
43+
else:
44+
# Fallback for older GitHub Actions runner versions
45+
print(f"::set-output name={name}::{value}")
46+
47+
def check_artifact_exists(artifact_name: str, token: str, repo: str, head_sha: str) -> bool:
48+
"""
49+
Check if an artifact exists in the repository for a specific commit.
50+
51+
Args:
52+
artifact_name: Name of the artifact to check
53+
token: GitHub token for authentication
54+
repo: Repository in format 'owner/repo'
55+
head_sha: Commit SHA to filter artifacts by
56+
57+
Returns:
58+
True if artifact exists, False otherwise
59+
"""
60+
headers = {
61+
'Authorization': f'token {token}',
62+
'Accept': 'application/vnd.github+json',
63+
'X-GitHub-Api-Version': '2022-11-28'
64+
}
65+
66+
# GitHub API endpoint to list artifacts for a repository
67+
url = f"https://api.github.com/repos/{repo}/actions/artifacts"
68+
69+
# Add query parameters - API will filter by artifact name
70+
params = {
71+
'per_page': 100,
72+
'name': artifact_name
73+
}
74+
75+
max_iterations = 100
76+
77+
try:
78+
page = 1
79+
while max_iterations > 0:
80+
params['page'] = page
81+
response = requests.get(url, headers=headers, params=params)
82+
response.raise_for_status()
83+
84+
artifacts_data = response.json()
85+
artifacts = artifacts_data.get('artifacts', [])
86+
total_count = artifacts_data.get('total_count', 0)
87+
88+
# No artifacts found at all
89+
if not artifacts:
90+
print(f"::debug::No artifacts found on page '{page}' with name {artifact_name} matching head_sha {head_sha}")
91+
break
92+
93+
# Check each artifact's head_sha since API filters by name already
94+
for artifact in artifacts:
95+
workflow_run = artifact.get('workflow_run', {})
96+
artifact_head_sha = workflow_run.get('head_sha')
97+
98+
if artifact_head_sha == head_sha:
99+
print(f"::debug::Artifact '{artifact_name}' found with ID {artifact.get('id')}, matching head_sha {head_sha}")
100+
return True
101+
else:
102+
print(f"::debug::Artifact '{artifact_name}' found but head_sha doesn't match (expected: {head_sha}, found: {artifact_head_sha})")
103+
104+
# Check if we've reached the last page
105+
# If we've retrieved all artifacts (page * per_page >= total_count) or
106+
# if this page has fewer artifacts than requested, we're done
107+
if page * params['per_page'] >= total_count or len(artifacts) < params['per_page']:
108+
break
109+
110+
page += 1
111+
max_iterations -= 1
112+
113+
print(f"::debug::Artifact '{artifact_name}' with head_sha '{head_sha}' not found")
114+
return False
115+
116+
except requests.exceptions.RequestException as e:
117+
print(f"::error::Failed to check artifact existence: {e}")
118+
sys.exit(1)
119+
except json.JSONDecodeError as e:
120+
print(f"::error::Failed to parse GitHub API response: {e}")
121+
sys.exit(1)
122+
123+
def main():
124+
"""Main function."""
125+
# Get inputs
126+
artifact_name = os.getenv('INPUT_ARTIFACT_NAME')
127+
if not artifact_name:
128+
print("::error::artifact_name input is required")
129+
sys.exit(1)
130+
131+
# Get environment variables
132+
token = get_github_token()
133+
repo = get_repository()
134+
head_sha = get_head_sha()
135+
136+
print(f"::debug::Checking for artifact '{artifact_name}' in repository '{repo}', head_sha '{head_sha}'")
137+
138+
# Check if artifact exists
139+
exists = check_artifact_exists(artifact_name, token, repo, head_sha)
140+
141+
# Set outputs
142+
set_output('exists', str(exists).lower())
143+
144+
print(f"::debug::Artifact {artifact_name} exists: {exists}")
145+
146+
if __name__ == '__main__':
147+
main()

.github/actions/deploy/action.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ runs:
5757
run: ./.github/resources/squid/deploy-squid.sh
5858

5959
- name: Download Docker Images
60-
uses: actions/download-artifact@v4
60+
uses: actions/download-artifact@v6
6161
with:
6262
path: "images_${{ github.sha }}"
6363

.github/workflows/image-builds-with-cache.yml

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ on:
1515
env:
1616
IMAGE_TAG: "latest"
1717
REGISTRY: "kind-registry:5000"
18+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
19+
GITHUB_SHA: ${{ github.sha }}
1820

1921
jobs:
2022
image-build-with-cache:
@@ -81,25 +83,15 @@ jobs:
8183
} >> "$GITHUB_OUTPUT"
8284
8385
# Check if the image tarball already exists or not, if yes, then skip building the image
84-
- name: Attempt to download the artifact
85-
uses: actions/download-artifact@v4
86+
- name: Check artifact exists
87+
uses: ./.github/actions/check-artifact-exists
8688
with:
87-
name: ${{ env.ARTIFACT_NAME }}
88-
path: ${{ env.ARTIFACTS_PATH }}
89-
continue-on-error: true
90-
id: artifact-download
91-
92-
# If the image tarball was successfully downloaded, then clean it up as the download should
93-
# happen when image tarballs are actually required for deployment (most likely in a new workflow/job)
94-
- name: Delete the artifact downloaded artifact
95-
if: ${{ steps.artifact-download.outcome == 'success' }}
96-
run: |
97-
rm -rf ${{ env.ARTIFACTS_PATH }}/${{ env.ARTIFACT_NAME }}
98-
shell: bash
89+
artifact_name: ${{ env.ARTIFACT_NAME }}
90+
id: artifact-check
9991

10092
- name: Set up Docker Buildx
10193
uses: docker/setup-buildx-action@v3
102-
if: ${{ steps.artifact-download.outcome == 'failure' }}
94+
if: ${{ steps.artifact-check.outputs.exists == 'false' }}
10395
id: setup-buildx
10496

10597
- name: Build and save Docker image
@@ -115,7 +107,7 @@ jobs:
115107

116108
- name: Rebuild Images in case of failure
117109
uses: docker/build-push-action@v5
118-
if: ${{ steps.save-image.outcome != 'success' }}
110+
if: ${{ steps.save-image.outcome != 'success' && steps.setup-buildx.outcome == 'success' }}
119111
id: rebuild
120112
with:
121113
context: ${{ matrix.context }}

.github/workflows/kfp-sdk-client-tests.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ on:
66

77
pull_request:
88
paths:
9-
- '.github/workflows/kfp-sdk-client-tests.yml.yml'
9+
- '.github/workflows/kfp-sdk-client-tests.yml'
1010
- '.github/actions/create-cluster/**'
1111
- '.github/resources/**'
1212
- 'sdk/python/**'

0 commit comments

Comments
 (0)