Skip to content

Commit a7119a8

Browse files
jhamonjhamon
authored
Move asyncio deps into extras (#446)
## Problem Rather than adding a new dependency for everyone, we want asyncio to be an extras install similar to how grpc is handled. ## Solution Adjust pyproject.toml to migrate aiohttp into an asyncio extras that will be installed like `pinecone[asyncio]`. Adjust test configuration to install the required dependencies. ## Type of Change - [x] Infrastructure change (CI configs, etc)
1 parent 2e44f78 commit a7119a8

File tree

9 files changed

+147
-115
lines changed

9 files changed

+147
-115
lines changed

‎.github/actions/setup-poetry/action.yml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,10 @@ inputs:
1313
description: 'Install typing dependencies (mypy, type stubs, etc)'
1414
required: true
1515
default: 'true'
16+
include_asyncio:
17+
description: 'Install asyncio dependencies'
18+
required: true
19+
default: 'false'
1620

1721
runs:
1822
using: 'composite'
@@ -26,8 +30,10 @@ runs:
2630
INCLUDE_GRPC: ${{ inputs.include_grpc }}
2731
INCLUDE_DEV: ${{ inputs.include_dev }}
2832
INCLUDE_TYPES: ${{ inputs.include_types }}
33+
INCLUDE_ASYNCIO: ${{ inputs.include_asyncio }}
2934
run: |
3035
GRPC_FLAG=$( [ "$INCLUDE_GRPC" = "true" ] && echo "--extras grpc" || echo "" )
36+
ASYNCIO_FLAG=$( [ "$INCLUDE_ASYNCIO" = "true" ] && echo "--extras asyncio" || echo "" )
3137
DEV_FLAG=$( [ "$INCLUDE_DEV" = "false" ] && echo "--without dev" || echo "" )
3238
TYPING_FLAG=$( [ "$INCLUDE_TYPES" = "true" ] && echo "--with types" || echo "" )
33-
poetry install $DEV_FLAG $TYPING_FLAG $GRPC_FLAG
39+
poetry install $DEV_FLAG $TYPING_FLAG $GRPC_FLAG $ASYNCIO_FLAG

‎.github/actions/test-asyncio/action.yaml renamed to ‎.github/actions/test-data-asyncio/action.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ runs:
3333
with:
3434
include_grpc: ${{ inputs.use_grpc }}
3535
include_dev: 'true'
36+
include_asyncio: 'true'
3637

3738
- name: Run data plane tests
3839
id: data-plane-asyncio-tests

‎.github/actions/test-dependency-asyncio-rest/action.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ runs:
2929
with:
3030
include_grpc: false
3131
include_types: false
32+
include_asyncio: true
3233

3334
- name: 'Install aiohttp ${{ matrix.aiohttp-version }}'
3435
run: 'poetry add aiohttp==${{ matrix.aiohttp-version }}'

‎.github/workflows/testing-integration-asyncio.yaml

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
name: "DB Integration: Asyncio"
2+
'on':
3+
workflow_call: {}
4+
5+
jobs:
6+
db-data-asyncio:
7+
name: db_data asyncio
8+
runs-on: ubuntu-latest
9+
strategy:
10+
fail-fast: false
11+
matrix:
12+
python_version:
13+
- 3.9
14+
- 3.13
15+
steps:
16+
- uses: actions/checkout@v4
17+
- name: Set up Python
18+
uses: actions/setup-python@v5
19+
with:
20+
python-version: ${{ matrix.python_version }}
21+
- name: Setup Poetry
22+
uses: ./.github/actions/setup-poetry
23+
with:
24+
include_dev: true
25+
include_asyncio: true
26+
- name: Run data plane tests
27+
id: data-plane-asyncio-tests
28+
shell: bash
29+
run: poetry run pytest tests/integration/data_asyncio -s -vv
30+
env:
31+
PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
32+
33+
db-control-asyncio:
34+
name: db_control asyncio
35+
runs-on: ubuntu-latest
36+
strategy:
37+
matrix:
38+
python_version:
39+
- 3.9
40+
- 3.12
41+
fail-fast: false
42+
steps:
43+
- uses: actions/checkout@v4
44+
- name: 'Set up Python ${{ matrix.python_version }}'
45+
uses: actions/setup-python@v5
46+
with:
47+
python-version: '${{ matrix.python_version }}'
48+
- name: Setup Poetry
49+
uses: ./.github/actions/setup-poetry
50+
with:
51+
include_asyncio: true
52+
include_dev: true
53+
- name: 'db_control asyncio'
54+
run: poetry run pytest tests/integration/control_asyncio -s -vv
55+
env:
56+
PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'

‎.github/workflows/testing-integration.yaml

Lines changed: 60 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -3,32 +3,37 @@ name: "Integration Tests"
33
workflow_call: {}
44

55
jobs:
6-
db-data-serverless:
7-
name: db_data rest
6+
7+
inference:
8+
name: Inference tests
89
runs-on: ubuntu-latest
9-
needs:
10-
- inference
1110
strategy:
12-
fail-fast: false
1311
matrix:
14-
python_version:
15-
- 3.9
16-
- 3.13
17-
use_grpc: [true, false]
12+
python_version: [3.9, 3.12]
1813
steps:
1914
- uses: actions/checkout@v4
20-
- uses: ./.github/actions/test-data-plane
15+
- name: 'Set up Python ${{ matrix.python_version }}'
16+
uses: actions/setup-python@v5
2117
with:
22-
python_version: '${{ matrix.python_version }}'
23-
use_grpc: '${{ matrix.use_grpc }}'
24-
metric: 'cosine'
25-
spec: '{ "serverless": { "region": "us-west-2", "cloud": "aws" }}'
18+
python-version: '${{ matrix.python_version }}'
19+
- name: Setup Poetry
20+
uses: ./.github/actions/setup-poetry
21+
with:
22+
include_asyncio: true
23+
- name: 'Run integration tests'
24+
run: poetry run pytest tests/integration/inference -s -vv
25+
env:
26+
PINECONE_DEBUG_CURL: 'true'
2627
PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
27-
freshness_timeout_seconds: 600
28-
skip_weird_id_tests: 'true'
2928

30-
db-data-asyncio:
31-
name: db_data asyncio
29+
30+
dependency-test-asyncio:
31+
uses: './.github/workflows/testing-integration-asyncio.yaml'
32+
secrets: inherit
33+
needs: inference
34+
35+
db-data-serverless:
36+
name: db_data rest
3237
runs-on: ubuntu-latest
3338
needs:
3439
- inference
@@ -38,48 +43,51 @@ jobs:
3843
python_version:
3944
- 3.9
4045
- 3.13
41-
use_grpc: [false, true]
46+
use_grpc: [true, false]
4247
steps:
4348
- uses: actions/checkout@v4
44-
- uses: ./.github/actions/test-asyncio
49+
- uses: ./.github/actions/test-data-plane
4550
with:
4651
python_version: '${{ matrix.python_version }}'
4752
use_grpc: '${{ matrix.use_grpc }}'
53+
metric: 'cosine'
4854
spec: '{ "serverless": { "region": "us-west-2", "cloud": "aws" }}'
4955
PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
5056
freshness_timeout_seconds: 600
57+
skip_weird_id_tests: 'true'
5158

52-
53-
db-control-rest-pod:
54-
name: db_control pod/collection tests
55-
runs-on: ubuntu-latest
56-
needs:
57-
- inference
58-
strategy:
59-
matrix:
60-
testConfig:
61-
- python-version: 3.9
62-
pod: { environment: 'us-east1-gcp'}
63-
- python-version: 3.13
64-
pod: { environment: 'us-east1-gcp'}
65-
fail-fast: false
66-
steps:
67-
- uses: actions/checkout@v4
68-
- name: 'Set up Python ${{ matrix.testConfig.python-version }}'
69-
uses: actions/setup-python@v5
70-
with:
71-
python-version: '${{ matrix.testConfig.python-version }}'
72-
- name: Setup Poetry
73-
uses: ./.github/actions/setup-poetry
74-
- name: 'Run integration tests (REST, prod)'
75-
run: poetry run pytest tests/integration/control/pod -s -v
76-
env:
77-
PINECONE_DEBUG_CURL: 'true'
78-
PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
79-
PINECONE_ENVIRONMENT: '${{ matrix.testConfig.pod.environment }}'
80-
GITHUB_BUILD_NUMBER: '${{ github.run_number }}-s-${{ matrix.testConfig.python-version}}'
81-
DIMENSION: 10
82-
METRIC: 'cosine'
59+
# db-control-rest-pod:
60+
# name: db_control pod/collection tests
61+
# runs-on: ubuntu-latest
62+
# needs:
63+
# - inference
64+
# strategy:
65+
# matrix:
66+
# testConfig:
67+
# - python-version: 3.9
68+
# pod: { environment: 'us-east1-gcp'}
69+
# - python-version: 3.13
70+
# pod: { environment: 'us-east1-gcp'}
71+
# fail-fast: false
72+
# steps:
73+
# - uses: actions/checkout@v4
74+
# - name: 'Set up Python ${{ matrix.testConfig.python-version }}'
75+
# uses: actions/setup-python@v5
76+
# with:
77+
# python-version: '${{ matrix.testConfig.python-version }}'
78+
# - name: Setup Poetry
79+
# uses: ./.github/actions/setup-poetry
80+
# with:
81+
# include_asyncio: true
82+
# - name: 'Run integration tests (REST)'
83+
# run: poetry run pytest tests/integration/control/pod -s -v
84+
# env:
85+
# PINECONE_DEBUG_CURL: 'true'
86+
# PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
87+
# PINECONE_ENVIRONMENT: '${{ matrix.testConfig.pod.environment }}'
88+
# GITHUB_BUILD_NUMBER: '${{ github.run_number }}-s-${{ matrix.testConfig.python-version}}'
89+
# DIMENSION: 10
90+
# METRIC: 'cosine'
8391

8492
db-control-rest-serverless:
8593
name: db_control serverless
@@ -104,55 +112,10 @@ jobs:
104112
python-version: '${{ matrix.testConfig.python-version }}'
105113
- name: Setup Poetry
106114
uses: ./.github/actions/setup-poetry
107-
- name: 'Run integration tests (REST, prod)'
115+
- name: 'Run integration tests (REST)'
108116
run: poetry run pytest tests/integration/control/serverless -s -vv
109117
env:
110118
PINECONE_DEBUG_CURL: 'true'
111119
PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
112120
SERVERLESS_CLOUD: '${{ matrix.testConfig.serverless.cloud }}'
113121
SERVERLESS_REGION: '${{ matrix.testConfig.serverless.region }}'
114-
115-
db-control-asyncio:
116-
name: db_control asyncio
117-
runs-on: ubuntu-latest
118-
needs:
119-
- inference
120-
strategy:
121-
matrix:
122-
python_version:
123-
- 3.9
124-
- 3.12
125-
fail-fast: false
126-
steps:
127-
- uses: actions/checkout@v4
128-
- name: 'Set up Python ${{ matrix.python_version }}'
129-
uses: actions/setup-python@v5
130-
with:
131-
python-version: '${{ matrix.python_version }}'
132-
- name: Setup Poetry
133-
uses: ./.github/actions/setup-poetry
134-
- name: 'Run integration tests (asyncio, prod)'
135-
run: poetry run pytest tests/integration/control_asyncio -s -vv
136-
env:
137-
PINECONE_DEBUG_CURL: 'true'
138-
PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'
139-
140-
inference:
141-
name: Inference tests
142-
runs-on: ubuntu-latest
143-
strategy:
144-
matrix:
145-
python_version: [3.9, 3.12]
146-
steps:
147-
- uses: actions/checkout@v4
148-
- name: 'Set up Python ${{ matrix.python_version }}'
149-
uses: actions/setup-python@v5
150-
with:
151-
python-version: '${{ matrix.python_version }}'
152-
- name: Setup Poetry
153-
uses: ./.github/actions/setup-poetry
154-
- name: 'Run integration tests'
155-
run: poetry run pytest tests/integration/inference -s -vv
156-
env:
157-
PINECONE_DEBUG_CURL: 'true'
158-
PINECONE_API_KEY: '${{ secrets.PINECONE_API_KEY }}'

‎.github/workflows/testing-unit.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ jobs:
2626
with:
2727
include_grpc: '${{ matrix.use_grpc }}'
2828
include_types: true
29+
include_asyncio: true
2930

3031
- name: mypy check
3132
run: poetry run mypy pinecone
@@ -54,6 +55,7 @@ jobs:
5455
with:
5556
include_grpc: '${{ matrix.use_grpc }}'
5657
include_types: false
58+
include_asyncio: true
5759

5860
- name: Run unit tests (REST)
5961
run: poetry run pytest --cov=pinecone --timeout=120 tests/unit

‎pinecone/openapi_support/rest_aiohttp.py

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,17 @@
33

44

55
class AiohttpRestClient(RestClientInterface):
6-
def __init__(self, configuration, pools_size=4, maxsize=None) -> None:
7-
import aiohttp
6+
def __init__(self, configuration) -> None:
7+
try:
8+
import aiohttp
9+
except ImportError:
10+
raise ImportError(
11+
"Additional dependencies are required to use Pinecone with asyncio. Include these extra dependencies in your project by installing `pinecone[asyncio]`."
12+
) from None
813

914
conn = aiohttp.TCPConnector()
1015
self._session = aiohttp.ClientSession(connector=conn)
1116

12-
# async def _cleanup(self):
13-
# if not self._session.closed:
14-
# await self._session.close()
15-
1617
async def close(self):
1718
await self._session.close()
1819

0 commit comments

Comments
 (0)