updating for pypi and CI

.github/workflows/main.yaml

@@ -0,0 +1,88 @@
+name: Context Data ETL Vector Pipeline CI
+
+on:
+  push:
+    branches: [ main, develop ]
+  pull_request:
+    branches: [ main, develop ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.8, 3.9, "3.10"]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+
+    - name: Run tests
+      run: |
+        pytest tests/
+
+    - name: Check code style
+      run: |
+        pip install flake8
+        flake8 .
+
+    - name: Type checking
+      run: |
+        pip install mypy
+        mypy etl_vector_pipeline
+
+  integration-test:
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+
+    - name: Run integration tests
+      run: |
+        pytest tests/integration
+
+  deploy:
+      runs-on: ubuntu-latest
+      needs: [build, integration-test]
+      if: github.ref == 'refs/heads/main' && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+      steps:
+      - uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install build twine
+
+      - name: Build package
+        run: python -m build
+
+      - name: Publish package
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
+        run: twine upload dist/*

.gitignore

@@ -166,6 +166,9 @@ vector_etl/config/*
 !vector_etl/config/
 !examples/
 
+# PyPI configuration
+.pypirc
+
 # Temporary test files
 vector_etl/tempfile_downloads/
 *_bkp.py

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=45", "wheel"]
+build-backend = "setuptools.build_meta"