feat/add-scheduler

.env.example

@@ -0,0 +1,14 @@
+# Python version
+PYTHON_VER_MIN_MAJOR=3
+PYTHON_VER_MIN_MINOR=9
+
+# Scheduler
+SCHEDULER_MAX_CONCURRENT_TASKS=10
+SCHEDULER_DEFAULT_TASK_TIMEOUT=60
+SCHEDULER_STATE_CHECK_INTERVAL=1
+SCHEDULER_CLEANUP_INTERVAL=3600
+SCHEDULER_STATE_FILE="scheduler_state.lock"
+
+# Pipeline 
+PIPELINE_MAX_PARALLEL=1
+PIPELINE_TIMEOUT=3600

.github/workflows/app-testing.yml

@@ -1,31 +1,100 @@
-name: build-and-test
+name: CI
 
 on:
   push:
-    branches: [ main ]
+    branches: [main]
   pull_request:
-    branches: [ main ]
+    branches: [main]
 
 jobs:
-  build:
+  check:
+
+    runs-on: ubuntu-latest
+
+    outputs:
+      has_changes: ${{ steps.check.outputs.has_changes }}
+      changed_files: ${{ steps.check.outputs.changed_files }}
+
+    steps:
+      # 1. Checkout the code from the repository
+    - name: Checkout Code
+      uses: actions/checkout@v4
+
+
+      # 2. Get a list of changed .py files
+    - name: Get changed files
+      id: changed-files
+      uses: tj-actions/changed-files@v44
+      with:
+        files: |
+          **.py
+
+      # 3. Set outputs for changed files
+    - name: Set change flag and files
+      id: check
+      run: |
+        if [ "${{ steps.changed-files.outputs.any_changed }}" == "true" ]; then
+          echo "has_changes=${{ steps.changed-files.outputs.any_changed }}" >> $GITHUB_OUTPUT
+          echo "changed_files=${{ steps.changed-files.outputs.all_changed_files }}" >> $GITHUB_OUTPUT
+        else
+          echo "has_changes=false" >> $GITHUB_OUTPUT
+          echo "changed_files=" >> $GITHUB_OUTPUT
+        fi
+
+
+  tests:
+    needs: [check]
+    if: ${{ needs.check.outputs.has_changes == 'true' }}
+
     runs-on: ubuntu-latest
+
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ['3.12', '3.13']
+
     steps:
-    - uses: actions/checkout@v3
+      # 1. Checkout the code from the repository
+    - name: Checkout Code
+      uses: actions/checkout@v4
+
+      # 2. Rename .env.example file to .env
+    - name: Rename env file
+      run: mv .env.example .env
+
+      # 3. Set up the Python environment
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v3
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+
+      # 4. Install uv
+    - name: Install uv
+      uses: astral-sh/setup-uv@v3
+
+      # 5. Install python
+    - name: Set up Python
+      run: uv python install
+
+      # 6. Install the project
+    - name: Install the project
+      run: uv sync --all-extras --dev
+
+      # 7. Install missing stub packages
+    - name: Install missing type stubs
+      run: |
+        uv run mypy --install-types --non-interactive .
+
+      # 8. Run Ruff Linter and show all errors
+    - name: Run Ruff Linter
       run: |
-        python -m pip install --upgrade pip
-        pip install flake8 pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Lint with flake8
+        echo "${{ needs.check.outputs.changed_files }}" | xargs uv run ruff check --output-format=github --config=pyproject.toml
+
+      # 9. Run Mypy Type Checking and show all errors
+    - name: Run Mypy Type Checking
+      run: |
+        echo "${{ needs.check.outputs.changed_files }}" | xargs uv run  mypy --config-file=pyproject.toml
+
+      # 10. Run Pytest
+    - name: Run Pytest
       run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=119 --statistics --config=setup.cfg
+        uv run pytest --maxfail=10 --disable-warnings --tb=short

.gitignore

@@ -1,4 +1,114 @@
 
+creds.ini
 .idea/
 __pycache__/
 *.pyc
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+.idea/**/aws.xml
+.idea/**/contentModel.xml
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+.idea/**/gradle.xml
+.idea/**/libraries
+cmake-build-*/
+.idea/**/mongoSettings.xml
+*.iws
+out/
+.idea_modules/
+atlassian-ide-plugin.xml
+.idea/replstate.xml
+.idea/sonarlint/
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+.idea/httpRequests
+.idea/caches/build_file_checksums.ser
+*.py[cod]
+*$py.class
+*.so
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+*.manifest
+*.spec
+pip-log.txt
+pip-delete-this-directory.txt
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+*.mo
+*.pot
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+instance/
+.webassets-cache
+.scrapy
+docs/_build/
+.pybuilder/
+target/
+.ipynb_checkpoints
+profile_default/
+ipython_config.py
+.pdm.toml
+__pypackages__/
+celerybeat-schedule
+celerybeat.pid
+*.sage.py
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+.spyderproject
+.spyproject
+.ropeproject
+/site
+.mypy_cache/
+.dmypy.json
+dmypy.json
+.pyre/
+.pytype/
+cython_debug/
+mypy_cache/
+ruff_cache/
+
+output.json

.pre-commit-config.yaml

@@ -0,0 +1,39 @@
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.6.0
+  hooks:
+  - id: check-yaml
+  - id: end-of-file-fixer
+  - id: trailing-whitespace
+  - id: no-commit-to-branch
+  - id: debug-statements
+  - id: check-json
+  - id: name-tests-test
+    args: [--pytest-test-first]
+  - id: pretty-format-json
+    args: [--autofix, --indent, '2']
+  - id: check-added-large-files
+
+- repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
+  rev: v2.14.0
+  hooks:
+  - id: pretty-format-toml
+    args: [--autofix, --indent, '2', --inline-comment-spaces, '2', --trailing-commas]
+  - id: pretty-format-yaml
+    args: [--autofix, --indent, '2']
+
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.6.9
+  hooks:
+  - id: ruff-format
+    args: [--config=pyproject.toml]
+  - id: ruff
+    args: [--config=pyproject.toml, --fix, --exit-non-zero-on-fix]
+
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v1.11.2
+  hooks:
+  - id: mypy
+    args: [--config-file=pyproject.toml, --install-types, --non-interactive]
+    additional_dependencies:
+    - pydantic

Makefile

@@ -0,0 +1,37 @@
+ONESHELL:
+
+.PHONY: env
+env:
+	@find . -name ".env.example" | while read file; do \
+		cp "$$file" "$$(dirname $$file)/.env"; \
+	done
+
+
+.PHONY: sync
+sync:
+	@uv sync --frozen --all-extras
+
+.PHONY: setup
+setup:
+	@curl -LsSf https://astral.sh/uv/install.sh | sh
+
+.PHONY: upd_hooks
+upd_hooks:
+	@pre-commit clean
+	@pre-commit install --install-hooks
+
+.PHONY: check
+check:
+	@git add .
+	@pre-commit run
+
+.PHONY: up
+up: env setup sync
+
+.PHONY: run
+run: sync env
+	@python -m src.main
+
+.PHONY: test
+test:
+	pytest