Skip to content

Commit d6ab9e5

Browse files
agredyaevagredyaev
authored
Merge pull request #4 from agredyaev/feat/add-scheduler-sc
Add scheduler
2 parents 21b4ed3 + 990c18f commit d6ab9e5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+3462
-109
lines changed

‎.env.example

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Python version
2+
PYTHON_MIN_MAJOR=3
3+
PYTHON_MIN_MINOR=9
4+
5+
# Scheduler
6+
SCHEDULER_MAX_CONCURRENT_TASKS=10
7+
SCHEDULER_DEFAULT_TASK_TIMEOUT=60
8+
SCHEDULER_STATE_CHECK_INTERVAL=1
9+
SCHEDULER_CLEANUP_INTERVAL=3600
10+
11+
# Pipeline
12+
PIPELINE_MAX_PARALLEL=1
13+
PIPELINE_TIMEOUT=3600
14+
15+
# State
16+
STATE_FILE_PATH="scheduler_state"
17+
STATE_VERSION=1
18+
STATE_CACHE_SIZE=100
19+
STATE_SAVE_INTERVAL=300

‎.github/workflows/app-testing.yml

Lines changed: 90 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,105 @@
1-
name: build-and-test
1+
name: CI
22

33
on:
44
push:
5-
branches: [ main ]
5+
branches: [main]
66
pull_request:
7-
branches: [ main ]
7+
branches: [main]
88

99
jobs:
10-
build:
10+
check:
11+
12+
runs-on: ubuntu-latest
13+
14+
outputs:
15+
has_changes: ${{ steps.check.outputs.has_changes }}
16+
changed_files: ${{ steps.check.outputs.changed_files }}
17+
18+
steps:
19+
# 1. Checkout the code from the repository
20+
- name: Checkout Code
21+
uses: actions/checkout@v4
22+
23+
24+
# 2. Get a list of changed .py files
25+
- name: Get changed files
26+
id: changed-files
27+
uses: tj-actions/changed-files@v44
28+
with:
29+
files: |
30+
**.py
31+
32+
# 3. Set outputs for changed files
33+
- name: Set change flag and files
34+
id: check
35+
run: |
36+
if [ "${{ steps.changed-files.outputs.any_changed }}" == "true" ]; then
37+
echo "has_changes=${{ steps.changed-files.outputs.any_changed }}" >> $GITHUB_OUTPUT
38+
echo "changed_files=${{ steps.changed-files.outputs.all_changed_files }}" >> $GITHUB_OUTPUT
39+
else
40+
echo "has_changes=false" >> $GITHUB_OUTPUT
41+
echo "changed_files=" >> $GITHUB_OUTPUT
42+
fi
43+
44+
45+
tests:
46+
needs: [check]
47+
if: ${{ needs.check.outputs.has_changes == 'true' }}
48+
1149
runs-on: ubuntu-latest
50+
1251
strategy:
1352
matrix:
14-
python-version: ["3.9", "3.10", "3.11"]
53+
python-version: ['3.12', '3.13']
54+
1555
steps:
16-
- uses: actions/checkout@v3
56+
# 1. Checkout the code from the repository
57+
- name: Checkout Code
58+
uses: actions/checkout@v4
59+
60+
# 2. Rename .env.example file to .env
61+
- name: Rename env file
62+
run: mv .env.example .env
63+
64+
# 3. Set up the Python environment
1765
- name: Set up Python ${{ matrix.python-version }}
18-
uses: actions/setup-python@v3
66+
uses: actions/setup-python@v5
1967
with:
2068
python-version: ${{ matrix.python-version }}
21-
- name: Install dependencies
69+
70+
# 4. Install uv
71+
- name: Install uv
72+
uses: astral-sh/setup-uv@v4
73+
74+
# 5. Install python
75+
- name: Set up Python
76+
run: uv python install
77+
78+
# 6. Install the project
79+
- name: Install the project
80+
run: uv sync --all-extras --dev
81+
82+
# 7.1 Install missing stub packages
83+
- name: Install missing type third-party stubs
2284
run: |
23-
python -m pip install --upgrade pip
24-
pip install flake8 pytest
25-
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
26-
- name: Lint with flake8
85+
uv pip install types-requests
86+
87+
# 7.2 Install missing stub packages
88+
- name: Install missing type stubs
89+
run: |
90+
uv run mypy --install-types --non-interactive .
91+
92+
# 8. Run Ruff Linter and show all errors
93+
- name: Run Ruff Linter
94+
run: |
95+
echo "${{ needs.check.outputs.changed_files }}" | xargs uv run ruff check --output-format=github --config=pyproject.toml
96+
97+
# 9. Run Mypy Type Checking and show all errors
98+
- name: Run Mypy Type Checking
99+
run: |
100+
echo "${{ needs.check.outputs.changed_files }}" | xargs uv run mypy --config-file=pyproject.toml
101+
102+
# 10. Run Pytest
103+
- name: Run Pytest
27104
run: |
28-
# stop the build if there are Python syntax errors or undefined names
29-
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
30-
# exit-zero treats all errors as warnings
31-
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=119 --statistics --config=setup.cfg
105+
uv run pytest --maxfail=10 --disable-warnings --tb=short

‎.gitignore

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,117 @@
11

2+
creds.ini
23
.idea/
34
__pycache__/
45
*.pyc
6+
.idea/**/workspace.xml
7+
.idea/**/tasks.xml
8+
.idea/**/usage.statistics.xml
9+
.idea/**/dictionaries
10+
.idea/**/shelf
11+
.idea/**/aws.xml
12+
.idea/**/contentModel.xml
13+
.idea/**/dataSources/
14+
.idea/**/dataSources.ids
15+
.idea/**/dataSources.local.xml
16+
.idea/**/sqlDataSources.xml
17+
.idea/**/dynamic.xml
18+
.idea/**/uiDesigner.xml
19+
.idea/**/dbnavigator.xml
20+
.idea/**/gradle.xml
21+
.idea/**/libraries
22+
cmake-build-*/
23+
.idea/**/mongoSettings.xml
24+
*.iws
25+
out/
26+
.idea_modules/
27+
atlassian-ide-plugin.xml
28+
.idea/replstate.xml
29+
.idea/sonarlint/
30+
com_crashlytics_export_strings.xml
31+
crashlytics.properties
32+
crashlytics-build.properties
33+
fabric.properties
34+
.idea/httpRequests
35+
.idea/caches/build_file_checksums.ser
36+
*.py[cod]
37+
*$py.class
38+
*.so
39+
.Python
40+
build/
41+
develop-eggs/
42+
dist/
43+
downloads/
44+
eggs/
45+
.eggs/
46+
lib/
47+
lib64/
48+
parts/
49+
sdist/
50+
var/
51+
wheels/
52+
share/python-wheels/
53+
*.egg-info/
54+
.installed.cfg
55+
*.egg
56+
MANIFEST
57+
*.manifest
58+
*.spec
59+
pip-log.txt
60+
pip-delete-this-directory.txt
61+
htmlcov/
62+
.tox/
63+
.nox/
64+
.coverage
65+
.coverage.*
66+
.cache
67+
nosetests.xml
68+
coverage.xml
69+
*.cover
70+
*.py,cover
71+
.hypothesis/
72+
.pytest_cache/
73+
cover/
74+
*.mo
75+
*.pot
76+
*.log
77+
local_settings.py
78+
db.sqlite3
79+
db.sqlite3-journal
80+
instance/
81+
.webassets-cache
82+
.scrapy
83+
docs/_build/
84+
.pybuilder/
85+
target/
86+
.ipynb_checkpoints
87+
profile_default/
88+
ipython_config.py
89+
.pdm.toml
90+
__pypackages__/
91+
celerybeat-schedule
92+
celerybeat.pid
93+
*.sage.py
94+
.env
95+
.venv
96+
env/
97+
venv/
98+
ENV/
99+
env.bak/
100+
venv.bak/
101+
.spyderproject
102+
.spyproject
103+
.ropeproject
104+
/site
105+
.mypy_cache/
106+
.dmypy.json
107+
dmypy.json
108+
.pyre/
109+
.pytype/
110+
cython_debug/
111+
mypy_cache/
112+
ruff_cache/
113+
114+
output.*
115+
tmp*
116+
test_state
117+
test_file.*

‎.pre-commit-config.yaml

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
repos:
2+
- repo: https://github.com/pre-commit/pre-commit-hooks
3+
rev: v5.0.0
4+
hooks:
5+
- id: check-yaml
6+
- id: end-of-file-fixer
7+
- id: trailing-whitespace
8+
- id: no-commit-to-branch
9+
- id: debug-statements
10+
- id: check-json
11+
- id: name-tests-test
12+
args: [--pytest-test-first]
13+
- id: pretty-format-json
14+
args: [--autofix, --indent, '2']
15+
- id: check-added-large-files
16+
17+
- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
18+
rev: v2.14.0
19+
hooks:
20+
- id: pretty-format-toml
21+
args: [--autofix, --indent, '2', --inline-comment-spaces, '2', --trailing-commas]
22+
- id: pretty-format-yaml
23+
args: [--autofix, --indent, '2']
24+
25+
- repo: https://github.com/astral-sh/ruff-pre-commit
26+
rev: v0.8.4
27+
hooks:
28+
- id: ruff-format
29+
args: [--config=pyproject.toml]
30+
- id: ruff
31+
args: [--config=pyproject.toml, --fix, --exit-non-zero-on-fix]
32+
33+
- repo: https://github.com/pre-commit/mirrors-mypy
34+
rev: v1.14.0
35+
hooks:
36+
- id: mypy
37+
args: [--config-file=pyproject.toml, --install-types, --non-interactive]
38+
additional_dependencies:
39+
- pydantic

‎Makefile

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
ONESHELL:
2+
3+
.PHONY: env
4+
env:
5+
@find . -name ".env.example" | while read file; do \
6+
cp "$$file" "$$(dirname $$file)/.env"; \
7+
done
8+
9+
10+
.PHONY: sync
11+
sync:
12+
@uv sync --frozen --all-extras
13+
14+
.PHONY: setup
15+
setup:
16+
@curl -LsSf https://astral.sh/uv/install.sh | sh
17+
18+
.PHONY: upd_hooks
19+
upd_hooks:
20+
@pre-commit clean
21+
@pre-commit install --install-hooks
22+
23+
.PHONY: check
24+
check:
25+
@git add .
26+
@pre-commit run
27+
28+
.PHONY: check-all
29+
check-all:
30+
@git add .
31+
@pre-commit run --all
32+
33+
.PHONY: up
34+
up: env setup sync
35+
36+
.PHONY: run
37+
run: sync env
38+
@python -m src.main
39+
40+
.PHONY: test
41+
test:
42+
pytest

0 commit comments

Comments
 (0)