chore: refactor python tests. no functional changes just moving into separate files

.github/workflows/primary.yml

@@ -27,6 +27,10 @@ permissions:
   pull-requests: read
   repository-projects: read
 
+env:
+  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+
 jobs:
   lint:
     runs-on: ubuntu-latest
@@ -48,7 +52,8 @@ jobs:
       - name: Check TS Lint
         run: pnpm biome ci . --organize-imports-enabled=false
         working-directory: typescript
-  test_node_generator:
+
+  test-typescript:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -66,21 +71,23 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: engine
-      - name: Install Node
-        run: pnpm install --frozen-lockfile
-        working-directory: engine/language_client_typescript
-      - name: Build Node
-        run: pnpm build:debug
-        working-directory: engine/language_client_typescript
-      - name: Install Node
-        run: pnpm install --frozen-lockfile
-        working-directory: integ-tests/typescript
-      - name: Test Node Generator
-        run: pnpm generate
-        working-directory: integ-tests/typescript
-      - name: Ensure No Changes
+      # Build TypeScript client
+      - name: Build TypeScript Client
+        run: |
+          cd engine/language_client_typescript
+          pnpm install --frozen-lockfile
+          pnpm build:debug
+      # Generate and test
+      - name: Generate and Test TypeScript
+        run: |
+          cd integ-tests/typescript
+          pnpm install --frozen-lockfile
+          pnpm generate
+          pnpm integ-tests:dotenv
+      - name: Verify No Changes
         run: git diff --exit-code
-  build:
+
+  test-rust:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -91,26 +98,45 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: engine
-      - name: Test Rust
-        # skip doctests & integration tests (the latter require OPENAI_API_KEY and ANTHROPIC_API_KEY to be set)
+      - name: Run Rust Tests
         run: cargo test --lib
         working-directory: engine
-  build-wasm:
+      # WASM tests
+      - name: Setup WASM Target
+        run: rustup target add wasm32-unknown-unknown
+      - name: Build WASM
+        run: cargo build --target=wasm32-unknown-unknown
+        working-directory: engine/baml-schema-wasm
+
+  test-python:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: jdx/mise-action@v2
       - uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: stable
-      - run: rustup target add wasm32-unknown-unknown
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: engine
-      - name: Build rust for wasm32
-        run: cargo build --target=wasm32-unknown-unknown
-        working-directory: engine/baml-schema-wasm
-  integ-tests:
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.8'
+          cache: 'poetry'
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+      - name: Run Python Integration Tests
+        run: |
+          cd integ-tests/python
+          poetry install
+          poetry run maturin develop --manifest-path ../../engine/language_client_python/Cargo.toml
+          poetry run baml-cli generate --from ../baml_src
+          poetry run pytest
+      - name: Verify No Changes
+        run: git diff --exit-code
+
+  test-ruby:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -121,8 +147,15 @@ jobs:
       - uses: Swatinem/rust-cache@v2
         with:
           workspaces: engine
-      - name: run python tests
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.2'
+          bundler-cache: true
+      - name: Run Ruby Integration Tests
         run: |
-          cd integ-tests/python
-          poetry install
-          ./run_tests.sh
+          cd integ-tests/ruby
+          bundle install
+          rake generate
+          rake test
+      - name: Verify No Changes
+        run: git diff --exit-code

.gitignore

@@ -164,3 +164,5 @@ web_modules/
 yarn-debug.log*
 yarn-error.log*
 yarn.lock
+.vars
+test-report.html

integ-tests/python/run_tests.sh

@@ -8,4 +8,4 @@ env -u CONDA_PREFIX poetry run maturin develop --manifest-path ../../engine/lang
 poetry run baml-cli generate --from ../baml_src
 
 # test_functions.py is excluded because it requires credentials
-poetry run pytest "$@" --ignore=tests/test_functions.py
+poetry run pytest

integ-tests/python/tests/__init__.py

@@ -0,0 +1 @@
+"""Integration tests for the BAML Python client."""

integ-tests/python/tests/caching_test.py

@@ -0,0 +1,40 @@
+import pytest
+import uuid
+import time
+from .test_setup import b
+
+@pytest.mark.asyncio
+async def test_caching():
+    story_idea = f"""
+In a futuristic world where dreams are a marketable asset and collective experience, an introverted and socially inept teenager named Alex realizes they have a unique and potent skill to not only observe but also alter the dreams of others. Initially excited by this newfound talent, Alex starts discreetly modifying the dreams of peers and relatives, aiding them in conquering fears, boosting self-esteem, or embarking on fantastical journeys. As Alex's abilities expand, so does their sway. They begin marketing exclusive dream experiences on the underground market, designing complex and captivating dreamscapes for affluent clients. However, the boundary between dream and reality starts to fade for those subjected to Alex's creations. Some clients find it difficult to distinguish between their genuine memories and the fabricated ones inserted by Alex's dream manipulation.
+
+Challenges emerge when a secretive government organization becomes aware of Alex's distinct talents. They propose Alex utilize their gift for "the greater good," suggesting uses in therapy, criminal reform, and even national defense. Concurrently, a covert resistance group contacts Alex, cautioning them about the risks of dream manipulation and the potential for widespread control and exploitation. Trapped between these conflicting forces, Alex must navigate a tangled web of moral dilemmas. They wrestle with issues of free will, the essence of consciousness, and the duty that comes with having influence over people's minds. As the repercussions of their actions ripple outward, impacting the lives of loved ones and strangers alike, Alex is compelled to face the true nature of their power and decide how—or if—it should be wielded.
+
+The narrative investigates themes of identity, the subconscious, the ethics of technology, and the power of creativity. It explores the possible outcomes of a world where our most intimate thoughts and experiences are no longer truly our own, and scrutinizes the fine line between aiding others and manipulating them for personal benefit or a perceived greater good. The story further delves into the societal ramifications of such abilities, questioning the moral limits of altering consciousness and the potential for misuse in a world where dreams can be commercialized. It challenges the reader to contemplate the impact of technology on personal freedom and the ethical duties of those who wield such power.
+
+As Alex's journey progresses, they meet various individuals whose lives have been influenced by their dream manipulations, each offering a distinct viewpoint on the ethical issues at hand. From a peer who gains newfound confidence to a wealthy client who becomes dependent on the dreamscapes, the ripple effects of Alex's actions are significant and extensive. The government agency's interest in Alex's abilities raises questions about the potential for state control and surveillance, while the resistance movement underscores the dangers of unchecked power and the necessity of protecting individual freedoms.
+
+Ultimately, Alex's story is one of self-discovery and moral reflection, as they must choose whether to use their abilities for personal gain, align with the government's vision of a controlled utopia, or join the resistance in their struggle for freedom and autonomy. The narrative encourages readers to reflect on the nature of reality, the boundaries of human experience, and the ethical implications of a world where dreams are no longer private sanctuaries but shared and manipulated commodities. It also examines the psychological impact on Alex, who must cope with the burden of knowing the intimate fears and desires of others, and the isolation that comes from being unable to share their own dreams without altering them.
+
+The story further investigates the technological progress that has made dream manipulation feasible, questioning the role of innovation in society and the potential for both advancement and peril. It considers the societal divide between those who can afford to purchase enhanced dream experiences and those who cannot, highlighting issues of inequality and access. As Alex becomes more ensnared in the web of their own making, they must confront the possibility that their actions could lead to unintended consequences, not just for themselves but for the fabric of society as a whole.
+
+In the end, Alex's journey is a cautionary tale about the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream.
+
+In conclusion, this story is a reflection on the power of dreams and the responsibilities that come with wielding such influence. It serves as a reminder of the importance of ethical considerations in the face of technological advancement and the need to balance innovation with humanity. The story leaves readers pondering the true cost of a world where dreams are no longer sacred, and the potential for both wonder and danger in the uncharted territories of the mind. But it's also a story about the power of imagination and the potential for change, even in a world where our deepest thoughts are no longer our own. And it's a story about the power of choice, and the importance of fighting for the freedom to dream.
+"""
+    rand = uuid.uuid4().hex
+    story_idea = rand + story_idea
+
+    start = time.time()
+    _ = await b.TestCaching(story_idea, "1. try to be funny")
+    duration = time.time() - start
+
+    start = time.time()
+    _ = await b.TestCaching(story_idea, "1. try to be funny")
+    duration2 = time.time() - start
+
+    print("Duration no caching: ", duration)
+    print("Duration with caching: ", duration2)
+
+    # Allow for some timing variation but ensure caching is significantly faster
+    assert duration2 < duration * 0.95, f"Expected second call ({duration2}s) to be at least 5% faster than first call ({duration}s)"

integ-tests/python/tests/constraints_test.py

@@ -0,0 +1,72 @@
+import pytest
+from assertpy import assert_that
+from baml_py import errors
+from .test_setup import b
+from ..baml_client.types import (
+    BlockConstraintForParam,
+    NestedBlockConstraintForParam,
+    MalformedConstraints2,
+)
+
+def all_succeeded(checks):
+    return all(check.status == "succeeded" for check in checks.values())
+
+@pytest.mark.asyncio
+async def test_constraints():
+    res = await b.PredictAge("Greg")
+    assert res.certainty.checks["unreasonably_certain"].status == "failed"
+    assert not (all_succeeded(res.certainty.checks))
+
+
+@pytest.mark.asyncio
+async def test_constraint_union_variant_checking():
+    res = await b.ExtractContactInfo(
+        "Reach me at [email protected], or 111-222-3333 if needed."
+    )
+    assert res.primary.value is not None
+    assert res.primary.value == "[email protected]"
+    assert res.secondary.value is not None
+    assert res.secondary.value == "111-222-3333"
+
+
+@pytest.mark.asyncio
+async def test_return_malformed_constraint():
+    with pytest.raises(errors.BamlError) as e:
+        res = await b.ReturnMalformedConstraints(1)
+        assert res.foo.value == 2
+        assert res.foo.checks["foo_check"].status == "failed"
+    assert "Failed to coerce value" in str(e)
+
+
+@pytest.mark.asyncio
+async def test_use_malformed_constraint():
+    with pytest.raises(errors.BamlError) as e:
+        res = await b.UseMalformedConstraints(MalformedConstraints2(foo=2))
+        assert res == 3
+    assert "object has no method named length" in str(e)
+
+
+@pytest.mark.asyncio
+async def test_block_constraints():
+    ret = await b.MakeBlockConstraint()
+    assert ret.checks["cross_field"].status == "failed"
+
+
+@pytest.mark.asyncio
+async def test_nested_block_constraints():
+    ret = await b.MakeNestedBlockConstraint()
+    print(ret)
+    assert ret.nbc.checks["cross_field"].status == "succeeded"
+
+
+@pytest.mark.asyncio
+async def test_block_constraint_arguments():
+    with pytest.raises(errors.BamlInvalidArgumentError) as e:
+        block_constraint = BlockConstraintForParam(bcfp=1, bcfp2="too long!")
+        await b.UseBlockConstraint(block_constraint)
+    assert "Failed assert: hi" in str(e)
+
+    with pytest.raises(errors.BamlInvalidArgumentError) as e:
+        nested_block_constraint = NestedBlockConstraintForParam(nbcfp=block_constraint)
+        await b.UseNestedBlockConstraint(nested_block_constraint)
+    assert "Failed assert: hi" in str(e)