Skip to content

Commit 1bd97b8

Browse files
minikinjmgilmanstevenj
authored
ci: Add project fields validator (#334)
* ci: add project-fields-validator * chore: update python and earthly * Update Earthfile * feat: update ProjectFieldsValidator * chore: refactor python code * Update README.md * Update main.py * chore: ci lint fixes * Update Earthfile * Update Earthfile * Update Earthfile * Update Earthfile * Update Earthfile * Update Earthfile * feat: add GitHub action * Update validate-project-fields.yml * Update validate-project-fields.yml * wip: clean up * chore: add GITHUB_TOKEN * Update Earthfile * Update validate-project-fields.yml * Update validate-project-fields.yml * Update validate-project-fields.yml * wip * wip * wip * Update validate-project-fields.yml * wip * wip: testing * wip: testing * wip: testing * wip: testing * wip: cleanup * Update main.py * Update main.py * Update main.py * Update validate-project-fields.yml * Feat(ci): validate project fields in prs and issues (#374) * feat(cat-ci): Make the action only use python standard libraries so its easier to use outside earthly * feat(python): Add feature to check stand alone python files (no 3rd party imports, no poetry) * feat(python): make check use stand alone python mode * feat(python): add local coce formater and linter check for python * feat(python): more edits for stand alone mode and debugging * fix(python): third parry imports check needs rust * fix(python): fix ci issues with changes made * fix(python): Fix CI issues found with the improved python tooling * fix(cat-ci): remove debug prints * fix(cat-ci): project action runner * fix(cat-ci): try action again * fix(cat-ci): Try and actually run the action now * fix(cat-ci): try a custom PAT secret * fix(cat-ci): Update token scope doc * fix(cat-ci): Rename GITHUB_TOKEN to make it distinct from other tokens * fix(cat-ci): secrets can't start with GITHUB_ * Update validate-project-fields.yml * Update validate-project-fields.yml * Update validate-project-fields.yml --------- Co-authored-by: Joshua Gilman <[email protected]> Co-authored-by: Steven Johnson <[email protected]>
1 parent 3a5dae3 commit 1bd97b8

File tree

25 files changed

+955
-148
lines changed

25 files changed

+955
-148
lines changed

.config/dictionaries/project.dic

+2
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ depgraph
1515
devenv
1616
dind
1717
dockerhub
18+
doseq
1819
doublecircle
1920
Earthfile
2021
Earthfiles
@@ -50,6 +51,7 @@ idents
5051
JDBC
5152
jorm
5253
jormungandr
54+
jsonlib
5355
junitreport
5456
Kroki
5557
kubeconfig
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
name: Validate Project Fields
2+
3+
on:
4+
pull_request:
5+
types:
6+
- opened
7+
- edited
8+
- synchronize
9+
- reopened
10+
- unassigned
11+
12+
permissions:
13+
contents: write
14+
pull-requests: write
15+
id-token: write
16+
repository-projects: write
17+
18+
concurrency:
19+
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
20+
cancel-in-progress: true
21+
22+
jobs:
23+
validate-project-fields:
24+
runs-on: ubuntu-latest
25+
env:
26+
# Needs a PAT Classic with (read:project)
27+
GITHUB_PROJECTS_PAT: ${{ secrets.PROJECTS_PAT }}
28+
GITHUB_REPOSITORY: "${{ github.repository }}"
29+
GITHUB_EVENT_NUMBER: "${{ github.event.number || '0' }}"
30+
PROJECT_NUMBER: 102
31+
steps:
32+
- name: Fetch Validation Script
33+
uses: actions/checkout@v4
34+
with:
35+
repository: input-output-hk/catalyst-ci
36+
ref: master
37+
sparse-checkout: |
38+
utilities/project-fields-validator/main.py
39+
sparse-checkout-cone-mode: false
40+
41+
- name: Set up Python
42+
uses: actions/setup-python@v5
43+
with:
44+
python-version: "3.13"
45+
46+
- name: Run Project Fields Validation
47+
if: always()
48+
continue-on-error: false
49+
run: utilities/project-fields-validator/main.py

Justfile

+11-1
Original file line numberDiff line numberDiff line change
@@ -16,5 +16,15 @@ check-spelling:
1616
earthly +clean-spelling-list
1717
earthly +check-spelling
1818

19+
20+
# Fix and Check Markdown files
21+
format-python-code:
22+
ruff check --select I --fix .
23+
ruff format .
24+
25+
# Fix and Check Markdown files
26+
lint-python:
27+
ruff check .
28+
1929
# Pre Push Checks - intended to be run by a git pre-push hook.
20-
pre-push: check-markdown check-spelling
30+
pre-push: check-markdown check-spelling format-python-code lint-python

earthly/docs/common/macros/include.py

+5-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import os
2-
import textwrap
32
import re
3+
import textwrap
4+
45

56
def inc_file(env, filename, start_line=0, end_line=None, indent=None):
67
"""
@@ -10,7 +11,7 @@ def inc_file(env, filename, start_line=0, end_line=None, indent=None):
1011
project.
1112
indent = number of spaces to indent every line but the first.
1213
"""
13-
14+
1415
try:
1516
full_filename = os.path.join(env.project_dir, filename)
1617

@@ -24,8 +25,8 @@ def inc_file(env, filename, start_line=0, end_line=None, indent=None):
2425
else:
2526
indent = " " * indent
2627
text = textwrap.indent(text, indent)
27-
text = text[len(indent):] # First line should not be indented at all.
28-
text = re.sub(r'\n$', '', text, count=1)
28+
text = text[len(indent) :] # First line should not be indented at all.
29+
text = re.sub(r"\n$", "", text, count=1)
2930
# print(text)
3031
return text
3132
except Exception as exc:

earthly/docs/dev/local.py

+4-4
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,13 @@
22

33
# cspell: words gmtime
44

5+
import argparse
56
import subprocess
7+
import sys
68
import time
9+
import urllib.request
710
import webbrowser
811
from dataclasses import dataclass, field
9-
import argparse
10-
import sys
11-
import urllib.request
1212

1313

1414
class ProcessRunError(Exception):
@@ -213,7 +213,7 @@ def main():
213213

214214
# Open the webpage in a browser (once)
215215
if not browsed:
216-
browsed=True
216+
browsed = True
217217
if not args.no_browser:
218218
webbrowser.open(f"http://localhost:{docs_container.exposed_port}")
219219

earthly/postgresql/scripts/std_checks.py

+3-4
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
11
#!/usr/bin/env python3
22

3+
import argparse
4+
35
import python.exec_manager as exec_manager
46
import python.vendor_files_check as vendor_files_check
5-
import argparse
67
import rich
7-
from rich import print
8-
import os
98

109
# This script is run inside the `check` stage for postgres database setup
1110
# to perform all high level non-compilation checks.
@@ -32,7 +31,7 @@ def main():
3231
# Force color output in CI
3332
rich.reconfigure(color_system="256")
3433

35-
parser = argparse.ArgumentParser(description="Postgres checks processing.")
34+
argparse.ArgumentParser(description="Postgres checks processing.")
3635

3736
results = exec_manager.Results("Postgres checks")
3837

earthly/postgresql/scripts/std_docs.py

+13-11
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,17 @@
22

33
# cspell: words dbmigrations dbhost dbuser dbuserpw Tsvg pgsql11
44

5-
from typing import Optional
6-
import python.exec_manager as exec_manager
7-
import python.db_ops as db_ops
85
import argparse
9-
import rich
10-
from rich import print
116
import os
127
import re
138
from textwrap import indent
149

10+
import python.db_ops as db_ops
11+
import python.exec_manager as exec_manager
12+
import rich
13+
from rich import print
14+
15+
1516
def process_sql_files(directory):
1617
file_pattern = r"V(\d+)__(\w+)\.sql"
1718
migrations = {}
@@ -32,11 +33,12 @@ def process_sql_files(directory):
3233
migrations[version] = {
3334
"version": version,
3435
"migration_name": migration_name,
35-
"sql_data": sql_data
36+
"sql_data": sql_data,
3637
}
3738

3839
return migrations, largest_version
3940

41+
4042
class Migrations:
4143
def __init__(self, args: argparse.Namespace):
4244
"""
@@ -73,6 +75,7 @@ def create_markdown_file(self, file_path):
7375

7476
print("Markdown file created successfully at: {}".format(file_path))
7577

78+
7679
def main():
7780
# Force color output in CI
7881
rich.reconfigure(color_system="256")
@@ -124,9 +127,7 @@ def main():
124127
f"-o docs/database_schema/ "
125128
)
126129
res = exec_manager.cli_run(
127-
schemaspy_cmd,
128-
name="Generate SchemaSpy Documentation",
129-
verbose=True
130+
schemaspy_cmd, name="Generate SchemaSpy Documentation", verbose=True
130131
)
131132
results.add(res)
132133

@@ -135,7 +136,7 @@ def main():
135136
exec_manager.cli_run(
136137
'echo "hide: true" > docs/database_schema/.pages',
137138
name="Create .pages file",
138-
verbose=True
139+
verbose=True,
139140
)
140141

141142
migrations.create_markdown_file("docs/migrations.md")
@@ -145,5 +146,6 @@ def main():
145146
if not results.ok():
146147
exit(1)
147148

149+
148150
if __name__ == "__main__":
149-
main()
151+
main()

earthly/python/Earthfile

+8-1
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,15 @@ python-base:
3535
# Adjust Poetry's configuration to prevent connection pool warnings.
3636
RUN poetry config installer.max-workers 10
3737

38+
# Extension we use needs rust.
39+
RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
40+
RUN echo 'source $HOME/.cargo/env' >> $HOME/.bashrc
41+
ENV PATH="/root/.cargo/bin:${PATH}"
42+
3843
# Install ruff for linting.
3944
RUN pip3 install ruff
4045
RUN pip3 install rich
46+
RUN pip3 install third-party-imports
4147

4248
# Universal build scripts we will always need and are not target dependent.
4349
COPY --dir scripts /scripts
@@ -58,9 +64,10 @@ BUILDER:
5864

5965
CHECK:
6066
FUNCTION
67+
ARG options
6168

6269
# Execute the check script
63-
RUN /scripts/std_checks.py
70+
RUN /scripts/std_checks.py $options
6471

6572
LINT_PYTHON:
6673
# Linting all Python code is done with ruff

earthly/python/scripts/std_checks.py

+82-12
Original file line numberDiff line numberDiff line change
@@ -4,44 +4,114 @@
44
import subprocess
55
import sys
66

7-
def check_pyproject_toml():
7+
8+
def check_pyproject_toml(stand_alone):
89
# Check if 'pyproject.toml' exists in the project root.
9-
if not os.path.isfile('pyproject.toml'):
10+
if not os.path.isfile("pyproject.toml"):
11+
if stand_alone:
12+
print("pyproject.toml check passed.")
13+
return True
14+
1015
print("Error: pyproject.toml not found.")
1116
return False
1217
else:
18+
if stand_alone:
19+
print("Error: pyproject.toml found in standalone python module.")
20+
return False
21+
1322
print("pyproject.toml check passed.")
1423
return True
15-
16-
def check_poetry_lock():
24+
25+
26+
def check_poetry_lock(stand_alone):
1727
# Check if 'poetry.lock' exists in the project root.
18-
if not os.path.isfile('poetry.lock'):
28+
if not os.path.isfile("poetry.lock"):
29+
if stand_alone:
30+
print("poetry.lock check passed.")
31+
return True
32+
1933
print("Error: poetry.lock not found.")
2034
return False
2135
else:
36+
if stand_alone:
37+
print("Error: poetry.lock found in stand alone module.")
38+
return False
39+
2240
print("poetry.lock check passed.")
2341
return True
2442

43+
44+
def check_lint_with_ruff():
45+
# Check Python code linting issues using 'ruff'.
46+
result = subprocess.run(["ruff", "check", "."], capture_output=True)
47+
if result.returncode != 0:
48+
print("Code linting issues found.")
49+
print(result.stdout.decode())
50+
return False
51+
else:
52+
print("Code linting check passed.")
53+
return True
54+
55+
2556
def check_code_format_with_ruff():
2657
# Check Python code formatting and linting issues using 'ruff'.
27-
result = subprocess.run(['ruff', 'check', '.'], capture_output=True)
58+
result = subprocess.run(["ruff", "format", "--check", "."], capture_output=True)
2859
if result.returncode != 0:
29-
print("Code formatting and linting issues found.")
60+
print("Code formatting issues found.")
3061
print(result.stdout.decode())
3162
return False
3263
else:
33-
print("Code formatting and linting check passed.")
64+
print("Code formatting check passed.")
3465
return True
3566

36-
def main():
67+
68+
def zero_third_party_packages_found(output):
69+
lines = output.split("\n") # Split the multiline string into individual lines
70+
71+
if len(lines) < 2:
72+
return False # The second line doesn't exist
73+
else:
74+
return lines[1].startswith("Found '0' third-party package imports")
75+
76+
77+
def check_no_third_party_imports():
78+
# Check No third party imports have been used
79+
result = subprocess.run(["third-party-imports", "."], capture_output=True)
80+
output = result.stdout.decode()
81+
82+
if result.returncode != 0 or not zero_third_party_packages_found(output):
83+
print("Checking third party imports failed.")
84+
print(output)
85+
return False
86+
else:
87+
print("Checking third party imports passed.")
88+
return True
89+
90+
91+
def main(stand_alone):
92+
if stand_alone:
93+
print(
94+
"Checking Standalone Python files (No third party imports or poetry project)"
95+
)
3796
checks_passed = True
3897
# Perform checks
39-
checks_passed &= check_pyproject_toml()
40-
checks_passed &= check_poetry_lock()
98+
99+
# These are true on python programs that require third party libraries, false otherwise
100+
checks_passed &= check_pyproject_toml(stand_alone)
101+
checks_passed &= check_poetry_lock(stand_alone)
102+
103+
# Always done
104+
checks_passed &= check_lint_with_ruff()
41105
checks_passed &= check_code_format_with_ruff()
42106

107+
# Only done if the code should be able to run without third part libraries
108+
if stand_alone:
109+
checks_passed &= check_no_third_party_imports()
110+
43111
if not checks_passed:
44112
sys.exit(1)
45113

114+
46115
if __name__ == "__main__":
47-
main()
116+
print(f"Current Working Directory: {os.getcwd()}")
117+
main("--stand-alone" in sys.argv[1:])

0 commit comments

Comments
 (0)