Skip to content

Commit 4157586

Browse files
stainless-app[bot]stainless-app[bot]
authored
release: 0.1.0-alpha.20 (#117)
* chore(internal): version bump * codegen metadata * chore(internal): avoid errors for isinstance checks on proxies * fix(package): support direct resource imports * codegen metadata * codegen metadata * feat(api): api update * feat(api): api update * chore(ci): upload sdks to package manager * release: 0.1.0-alpha.20 --------- Co-authored-by: stainless-app[bot] <142633134+stainless-app[bot]@users.noreply.github.com>
1 parent 0689f69 commit 4157586

File tree

18 files changed

+161
-45
lines changed

18 files changed

+161
-45
lines changed

‎.github/workflows/ci.yml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,3 +29,27 @@ jobs:
2929

3030
- name: Run lints
3131
run: ./scripts/lint
32+
33+
upload:
34+
if: github.repository == 'stainless-sdks/codex-python'
35+
timeout-minutes: 10
36+
name: upload
37+
permissions:
38+
contents: read
39+
id-token: write
40+
runs-on: depot-ubuntu-24.04
41+
steps:
42+
- uses: actions/checkout@v4
43+
44+
- name: Get GitHub OIDC Token
45+
id: github-oidc
46+
uses: actions/github-script@v6
47+
with:
48+
script: core.setOutput('github_token', await core.getIDToken());
49+
50+
- name: Upload tarball
51+
env:
52+
URL: https://pkg.stainless.com/s
53+
AUTH: ${{ steps.github-oidc.outputs.github_token }}
54+
SHA: ${{ github.sha }}
55+
run: ./scripts/utils/upload-artifact.sh

‎.release-please-manifest.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
".": "0.1.0-alpha.19"
2+
".": "0.1.0-alpha.20"
33
}

‎.stats.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
configured_endpoints: 44
2-
openapi_spec_hash: 97719fe7ae4c641a5a020dd21f2978dd
2+
openapi_spec_hash: 9d81a4b0eca6d3629ba9d5432a65655c
33
config_hash: 659f65b6ccf5612986f920f7f9abbcb5

‎CHANGELOG.md

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,26 @@
11
# Changelog
22

3+
## 0.1.0-alpha.20 (2025-05-15)
4+
5+
Full Changelog: [v0.1.0-alpha.19...v0.1.0-alpha.20](https://github.com/cleanlab/codex-python/compare/v0.1.0-alpha.19...v0.1.0-alpha.20)
6+
7+
### Features
8+
9+
* **api:** api update ([2e74162](https://github.com/cleanlab/codex-python/commit/2e741628a380d0fefe117f80bb3796b111575df3))
10+
* **api:** api update ([9e85827](https://github.com/cleanlab/codex-python/commit/9e85827e0b1a58011a8ead15c695cb175744325a))
11+
12+
13+
### Bug Fixes
14+
15+
* **package:** support direct resource imports ([09066c8](https://github.com/cleanlab/codex-python/commit/09066c8bf38b23fd3d902b42c4f4f769161b0e2e))
16+
17+
18+
### Chores
19+
20+
* **ci:** upload sdks to package manager ([6594b48](https://github.com/cleanlab/codex-python/commit/6594b48736ea79e7f9457cb3b47abfa17618565b))
21+
* **internal:** avoid errors for isinstance checks on proxies ([a1d7faf](https://github.com/cleanlab/codex-python/commit/a1d7fafa46e9100a4d29c46b48919025b26a0cfa))
22+
* **internal:** version bump ([971e28d](https://github.com/cleanlab/codex-python/commit/971e28dd483b3f2d38094f368baebd5eb0906e2c))
23+
324
## 0.1.0-alpha.19 (2025-05-07)
425

526
Full Changelog: [v0.1.0-alpha.18...v0.1.0-alpha.19](https://github.com/cleanlab/codex-python/compare/v0.1.0-alpha.18...v0.1.0-alpha.19)

‎pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
[project]
22
name = "codex-sdk"
3-
version = "0.1.0-alpha.19"
3+
version = "0.1.0-alpha.20"
44
description = "Internal SDK used within cleanlab-codex package. Refer to https://pypi.org/project/cleanlab-codex/ instead."
55
dynamic = ["readme"]
66
license = "MIT"

‎scripts/utils/upload-artifact.sh

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/usr/bin/env bash
2+
set -exuo pipefail
3+
4+
RESPONSE=$(curl -X POST "$URL" \
5+
-H "Authorization: Bearer $AUTH" \
6+
-H "Content-Type: application/json")
7+
8+
SIGNED_URL=$(echo "$RESPONSE" | jq -r '.url')
9+
10+
if [[ "$SIGNED_URL" == "null" ]]; then
11+
echo -e "\033[31mFailed to get signed URL.\033[0m"
12+
exit 1
13+
fi
14+
15+
UPLOAD_RESPONSE=$(tar -cz . | curl -v -X PUT \
16+
-H "Content-Type: application/gzip" \
17+
--data-binary @- "$SIGNED_URL" 2>&1)
18+
19+
if echo "$UPLOAD_RESPONSE" | grep -q "HTTP/[0-9.]* 200"; then
20+
echo -e "\033[32mUploaded build to Stainless storage.\033[0m"
21+
echo -e "\033[32mInstallation: npm install 'https://pkg.stainless.com/s/codex-python/$SHA'\033[0m"
22+
else
23+
echo -e "\033[31mFailed to upload artifact.\033[0m"
24+
exit 1
25+
fi

‎src/codex/__init__.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
22

3+
import typing as _t
4+
35
from . import types
46
from ._types import NOT_GIVEN, Omit, NoneType, NotGiven, Transport, ProxiesTypes
57
from ._utils import file_from_path
@@ -80,6 +82,9 @@
8082
"DefaultAsyncHttpxClient",
8183
]
8284

85+
if not _t.TYPE_CHECKING:
86+
from ._utils._resources_proxy import resources as resources
87+
8388
_setup_logging()
8489

8590
# Update the __module__ attribute for exported symbols so that

‎src/codex/_utils/_proxy.py

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,10 @@ def __dir__(self) -> Iterable[str]:
4646
@property # type: ignore
4747
@override
4848
def __class__(self) -> type: # pyright: ignore
49-
proxied = self.__get_proxied__()
49+
try:
50+
proxied = self.__get_proxied__()
51+
except Exception:
52+
return type(self)
5053
if issubclass(type(proxied), LazyProxy):
5154
return type(proxied)
5255
return proxied.__class__

‎src/codex/_utils/_resources_proxy.py

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
from __future__ import annotations
2+
3+
from typing import Any
4+
from typing_extensions import override
5+
6+
from ._proxy import LazyProxy
7+
8+
9+
class ResourcesProxy(LazyProxy[Any]):
10+
"""A proxy for the `codex.resources` module.
11+
12+
This is used so that we can lazily import `codex.resources` only when
13+
needed *and* so that users can just import `codex` and reference `codex.resources`
14+
"""
15+
16+
@override
17+
def __load__(self) -> Any:
18+
import importlib
19+
20+
mod = importlib.import_module("codex.resources")
21+
return mod
22+
23+
24+
resources = ResourcesProxy().__as_proxied__()

‎src/codex/_version.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
# File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
22

33
__title__ = "codex"
4-
__version__ = "0.1.0-alpha.19" # x-release-please-version
4+
__version__ = "0.1.0-alpha.20" # x-release-please-version

‎src/codex/resources/projects/clusters.py

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,9 @@ def list(
5050
self,
5151
project_id: str,
5252
*,
53-
eval_issue_types: List[Literal["hallucination", "search_failure", "unhelpful", "difficult_query"]]
53+
eval_issue_types: List[
54+
Literal["hallucination", "search_failure", "unhelpful", "difficult_query", "unsupported"]
55+
]
5456
| NotGiven = NOT_GIVEN,
5557
instruction_adherence_failure: Optional[Literal["html_format", "content_structure"]] | NotGiven = NOT_GIVEN,
5658
limit: int | NotGiven = NOT_GIVEN,
@@ -177,7 +179,9 @@ def list(
177179
self,
178180
project_id: str,
179181
*,
180-
eval_issue_types: List[Literal["hallucination", "search_failure", "unhelpful", "difficult_query"]]
182+
eval_issue_types: List[
183+
Literal["hallucination", "search_failure", "unhelpful", "difficult_query", "unsupported"]
184+
]
181185
| NotGiven = NOT_GIVEN,
182186
instruction_adherence_failure: Optional[Literal["html_format", "content_structure"]] | NotGiven = NOT_GIVEN,
183187
limit: int | NotGiven = NOT_GIVEN,

‎src/codex/resources/projects/projects.py

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -426,8 +426,8 @@ def validate(
426426
query: str,
427427
response: str,
428428
use_llm_matching: bool | NotGiven = NOT_GIVEN,
429-
bad_response_thresholds: project_validate_params.BadResponseThresholds | NotGiven = NOT_GIVEN,
430429
constrain_outputs: Optional[List[str]] | NotGiven = NOT_GIVEN,
430+
custom_eval_thresholds: Optional[Dict[str, float]] | NotGiven = NOT_GIVEN,
431431
custom_metadata: Optional[object] | NotGiven = NOT_GIVEN,
432432
eval_scores: Optional[Dict[str, float]] | NotGiven = NOT_GIVEN,
433433
options: Optional[project_validate_params.Options] | NotGiven = NOT_GIVEN,
@@ -451,10 +451,13 @@ def validate(
451451
query will be recorded in the project for SMEs to answer.
452452
453453
Args:
454+
custom_eval_thresholds: Optional custom thresholds for specific evals. Keys should match with the keys
455+
in the `eval_scores` dictionary.
456+
454457
custom_metadata: Arbitrary metadata supplied by the user/system
455458
456-
eval_scores: Evaluation scores to use for flagging a response as bad. If not provided, TLM
457-
will be used to generate scores.
459+
eval_scores: Scores assessing different aspects of the RAG system. If not provided, TLM will
460+
be used to generate scores.
458461
459462
options: Typed dict of advanced configuration options for the Trustworthy Language Model.
460463
Many of these configurations are determined by the quality preset selected
@@ -575,8 +578,8 @@ def validate(
575578
"prompt": prompt,
576579
"query": query,
577580
"response": response,
578-
"bad_response_thresholds": bad_response_thresholds,
579581
"constrain_outputs": constrain_outputs,
582+
"custom_eval_thresholds": custom_eval_thresholds,
580583
"custom_metadata": custom_metadata,
581584
"eval_scores": eval_scores,
582585
"options": options,
@@ -967,8 +970,8 @@ async def validate(
967970
query: str,
968971
response: str,
969972
use_llm_matching: bool | NotGiven = NOT_GIVEN,
970-
bad_response_thresholds: project_validate_params.BadResponseThresholds | NotGiven = NOT_GIVEN,
971973
constrain_outputs: Optional[List[str]] | NotGiven = NOT_GIVEN,
974+
custom_eval_thresholds: Optional[Dict[str, float]] | NotGiven = NOT_GIVEN,
972975
custom_metadata: Optional[object] | NotGiven = NOT_GIVEN,
973976
eval_scores: Optional[Dict[str, float]] | NotGiven = NOT_GIVEN,
974977
options: Optional[project_validate_params.Options] | NotGiven = NOT_GIVEN,
@@ -992,10 +995,13 @@ async def validate(
992995
query will be recorded in the project for SMEs to answer.
993996
994997
Args:
998+
custom_eval_thresholds: Optional custom thresholds for specific evals. Keys should match with the keys
999+
in the `eval_scores` dictionary.
1000+
9951001
custom_metadata: Arbitrary metadata supplied by the user/system
9961002
997-
eval_scores: Evaluation scores to use for flagging a response as bad. If not provided, TLM
998-
will be used to generate scores.
1003+
eval_scores: Scores assessing different aspects of the RAG system. If not provided, TLM will
1004+
be used to generate scores.
9991005
10001006
options: Typed dict of advanced configuration options for the Trustworthy Language Model.
10011007
Many of these configurations are determined by the quality preset selected
@@ -1116,8 +1122,8 @@ async def validate(
11161122
"prompt": prompt,
11171123
"query": query,
11181124
"response": response,
1119-
"bad_response_thresholds": bad_response_thresholds,
11201125
"constrain_outputs": constrain_outputs,
1126+
"custom_eval_thresholds": custom_eval_thresholds,
11211127
"custom_metadata": custom_metadata,
11221128
"eval_scores": eval_scores,
11231129
"options": options,

‎src/codex/types/project_validate_params.py

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
from .._utils import PropertyInfo
99

10-
__all__ = ["ProjectValidateParams", "BadResponseThresholds", "Options"]
10+
__all__ = ["ProjectValidateParams", "Options"]
1111

1212

1313
class ProjectValidateParams(TypedDict, total=False):
@@ -21,15 +21,19 @@ class ProjectValidateParams(TypedDict, total=False):
2121

2222
use_llm_matching: bool
2323

24-
bad_response_thresholds: BadResponseThresholds
25-
2624
constrain_outputs: Optional[List[str]]
2725

26+
custom_eval_thresholds: Optional[Dict[str, float]]
27+
"""Optional custom thresholds for specific evals.
28+
29+
Keys should match with the keys in the `eval_scores` dictionary.
30+
"""
31+
2832
custom_metadata: Optional[object]
2933
"""Arbitrary metadata supplied by the user/system"""
3034

3135
eval_scores: Optional[Dict[str, float]]
32-
"""Evaluation scores to use for flagging a response as bad.
36+
"""Scores assessing different aspects of the RAG system.
3337
3438
If not provided, TLM will be used to generate scores.
3539
"""
@@ -139,16 +143,6 @@ class ProjectValidateParams(TypedDict, total=False):
139143
x_stainless_package_version: Annotated[str, PropertyInfo(alias="x-stainless-package-version")]
140144

141145

142-
class BadResponseThresholds(TypedDict, total=False):
143-
context_sufficiency: Optional[float]
144-
145-
query_ease: Optional[float]
146-
147-
response_helpfulness: Optional[float]
148-
149-
trustworthiness: Optional[float]
150-
151-
152146
class Options(TypedDict, total=False):
153147
custom_eval_criteria: Iterable[object]
154148

‎src/codex/types/project_validate_response.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99

1010
class EvalScores(BaseModel):
11-
is_bad: bool
11+
failed: bool
1212

1313
score: Optional[float] = None
1414

@@ -18,7 +18,7 @@ class EvalScores(BaseModel):
1818
class ProjectValidateResponse(BaseModel):
1919
eval_scores: Dict[str, EvalScores]
2020
"""
21-
Evaluation scores for the original response along with a boolean flag, `is_bad`,
21+
Evaluation scores for the original response along with a boolean flag, `failed`,
2222
indicating whether the score is below the threshold.
2323
"""
2424

‎src/codex/types/projects/cluster_list_params.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010

1111
class ClusterListParams(TypedDict, total=False):
12-
eval_issue_types: List[Literal["hallucination", "search_failure", "unhelpful", "difficult_query"]]
12+
eval_issue_types: List[Literal["hallucination", "search_failure", "unhelpful", "difficult_query", "unsupported"]]
1313

1414
instruction_adherence_failure: Optional[Literal["html_format", "content_structure"]]
1515

‎src/codex/types/projects/entry_notify_sme_params.py

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,13 @@ class EntryNotifySmeParams(TypedDict, total=False):
1818
class ViewContext(TypedDict, total=False):
1919
page: Required[int]
2020

21-
filter: Literal["unanswered", "answered", "all", "hallucination", "search_failure", "unhelpful", "difficult_query"]
21+
filter: Literal[
22+
"unanswered",
23+
"answered",
24+
"all",
25+
"hallucination",
26+
"search_failure",
27+
"unhelpful",
28+
"difficult_query",
29+
"unsupported",
30+
]

‎tests/api_resources/test_projects.py

Lines changed: 2 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -444,13 +444,8 @@ def test_method_validate_with_all_params(self, client: Codex) -> None:
444444
query="query",
445445
response="response",
446446
use_llm_matching=True,
447-
bad_response_thresholds={
448-
"context_sufficiency": 0,
449-
"query_ease": 0,
450-
"response_helpfulness": 0,
451-
"trustworthiness": 0,
452-
},
453447
constrain_outputs=["string"],
448+
custom_eval_thresholds={"foo": 0},
454449
custom_metadata={},
455450
eval_scores={"foo": 0},
456451
options={
@@ -944,13 +939,8 @@ async def test_method_validate_with_all_params(self, async_client: AsyncCodex) -
944939
query="query",
945940
response="response",
946941
use_llm_matching=True,
947-
bad_response_thresholds={
948-
"context_sufficiency": 0,
949-
"query_ease": 0,
950-
"response_helpfulness": 0,
951-
"trustworthiness": 0,
952-
},
953942
constrain_outputs=["string"],
943+
custom_eval_thresholds={"foo": 0},
954944
custom_metadata={},
955945
eval_scores={"foo": 0},
956946
options={

‎tests/test_utils/test_proxy.py

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,14 @@ def test_recursive_proxy() -> None:
2121
assert dir(proxy) == []
2222
assert type(proxy).__name__ == "RecursiveLazyProxy"
2323
assert type(operator.attrgetter("name.foo.bar.baz")(proxy)).__name__ == "RecursiveLazyProxy"
24+
25+
26+
def test_isinstance_does_not_error() -> None:
27+
class AlwaysErrorProxy(LazyProxy[Any]):
28+
@override
29+
def __load__(self) -> Any:
30+
raise RuntimeError("Mocking missing dependency")
31+
32+
proxy = AlwaysErrorProxy()
33+
assert not isinstance(proxy, dict)
34+
assert isinstance(proxy, LazyProxy)

0 commit comments

Comments
 (0)