Introduce unvectorized pytest marker

Author: honno

array_api_tests/conftest.py

@@ -77,6 +77,10 @@ def pytest_configure(config):
         "markers",
         "min_version(api_version): run when greater or equal to api_version",
     )
+    config.addinivalue_line(
+        "markers",
+        "unvectorized: asserts against values via element-wise iteration (not performative!)",
+    )
     # Hypothesis
     hypothesis_max_examples = config.getoption("--hypothesis-max-examples")
     disable_deadline = config.getoption("--hypothesis-disable-deadline")
@@ -104,6 +108,9 @@ def xp_has_ext(ext: str) -> bool:
 
 
 def pytest_collection_modifyitems(config, items):
+    # 1. Prepare for iterating over items
+    # -----------------------------------
+
     skips_file = skips_path = config.getoption('--skips-file')
     if skips_file is None:
         skips_file = Path(__file__).parent / "skips.txt"
@@ -139,6 +146,9 @@ def pytest_collection_modifyitems(config, items):
     disabled_exts = config.getoption("--disable-extension")
     disabled_dds = config.getoption("--disable-data-dependent-shapes")
 
+    # 2. Iterate through items and apply markers accordingly
+    # ------------------------------------------------------
+
     for item in items:
         markers = list(item.iter_markers())
         # skip if specified in skips file
@@ -182,6 +192,9 @@ def pytest_collection_modifyitems(config, items):
                     )
                 )
 
+    # 3. Warn on bad skipped/xfailed ids
+    # ----------------------------------
+
     bad_ids_end_msg = (
         "Note the relevant tests might not of been collected by pytest, or "
         "another specified id might have already matched a test."
@@ -203,4 +216,4 @@ def pytest_collection_modifyitems(config, items):
             f"{f_bad_ids}\n"
             f"(xfails file: {xfails_file})\n"
             f"{bad_ids_end_msg}"
-        )
+        )

array_api_tests/test_array_object.py

@@ -105,6 +105,7 @@ def test_getitem(shape, dtype, data):
         ph.assert_array_elements("__getitem__", out=out, expected=expected)
 
 
+@pytest.mark.unvectorized
 @given(
     shape=hh.shapes(),
     dtypes=hh.oneway_promotable_dtypes(dh.all_dtypes),
@@ -152,6 +153,7 @@ def test_setitem(shape, dtypes, data):
         )
 
 
+@pytest.mark.unvectorized
 @pytest.mark.data_dependent_shapes
 @given(hh.shapes(), st.data())
 def test_getitem_masking(shape, data):
@@ -197,6 +199,7 @@ def test_getitem_masking(shape, data):
                 )
 
 
+@pytest.mark.unvectorized
 @given(hh.shapes(), st.data())
 def test_setitem_masking(shape, data):
     x = data.draw(hh.arrays(xps.scalar_dtypes(), shape=shape), label="x")

array_api_tests/test_indexing_functions.py

@@ -10,6 +10,7 @@
 from . import xps
 
 
+@pytest.mark.unvectorized
 @pytest.mark.min_version("2022.12")
 @given(
     x=hh.arrays(xps.scalar_dtypes(), hh.shapes(min_dims=1, min_side=1)),

array_api_tests/test_manipulation_functions.py

@@ -119,6 +119,7 @@ def test_concat(dtypes, base_shape, data):
                     )
 
 
+@pytest.mark.unvectorized
 @given(
     x=hh.arrays(dtype=xps.scalar_dtypes(), shape=shared_shapes()),
     axis=shared_shapes().flatmap(
@@ -147,6 +148,7 @@ def test_expand_dims(x, axis):
     )
 
 
+@pytest.mark.unvectorized
 @given(
     x=hh.arrays(
         dtype=xps.scalar_dtypes(), shape=hh.shapes(min_side=1).filter(lambda s: 1 in s)
@@ -184,6 +186,7 @@ def test_squeeze(x, data):
     assert_array_ndindex("squeeze", x, x_indices=sh.ndindex(x.shape), out=out, out_indices=sh.ndindex(out.shape))
 
 
+@pytest.mark.unvectorized
 @given(
     x=hh.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes()),
     data=st.data(),
@@ -208,6 +211,7 @@ def test_flip(x, data):
                              out_indices=reverse_indices, kw=kw)
 
 
+@pytest.mark.unvectorized
 @given(
     x=hh.arrays(dtype=xps.scalar_dtypes(), shape=shared_shapes(min_dims=1)),
     axes=shared_shapes(min_dims=1).flatmap(
@@ -248,6 +252,7 @@ def reshape_shapes(draw, shape):
     return tuple(rshape)
 
 
+@pytest.mark.unvectorized
 @pytest.mark.skip("flaky")  # TODO: fix!
 @given(
     x=hh.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes(max_side=MAX_SIDE)),
@@ -280,6 +285,7 @@ def roll_ndindex(shape: Shape, shifts: Tuple[int], axes: Tuple[int]) -> Iterator
         yield tuple((i + sh) % si for i, sh, si in zip(idx, all_shifts, shape))
 
 
+@pytest.mark.unvectorized
 @given(hh.arrays(dtype=xps.scalar_dtypes(), shape=shared_shapes()), st.data())
 def test_roll(x, data):
     shift_strat = st.integers(-hh.MAX_ARRAY_SIZE, hh.MAX_ARRAY_SIZE)
@@ -319,6 +325,7 @@ def test_roll(x, data):
         assert_array_ndindex("roll", x, x_indices=sh.ndindex(x.shape), out=out, out_indices=shifted_indices, kw=kw)
 
 
+@pytest.mark.unvectorized
 @given(
     shape=shared_shapes(min_dims=1),
     dtypes=hh.mutually_promotable_dtypes(None),

array_api_tests/test_operators_and_elementwise_functions.py

@@ -22,6 +22,9 @@
 from .typing import Array, DataType, Param, Scalar, ScalarType, Shape
 
 
+pytestmark = pytest.mark.unvectorized
+
+
 def all_integer_dtypes() -> st.SearchStrategy[DataType]:
     """Returns a strategy for signed and unsigned integer dtype objects."""
     return xps.unsigned_integer_dtypes() | xps.integer_dtypes()

array_api_tests/test_searching_functions.py

@@ -12,6 +12,9 @@
 from . import xps
 
 
+pytestmark = pytest.mark.unvectorized
+
+
 @given(
     x=hh.arrays(
         dtype=xps.real_dtypes(),

array_api_tests/test_set_functions.py

@@ -13,7 +13,7 @@
 from . import shape_helpers as sh
 from . import xps
 
-pytestmark = pytest.mark.data_dependent_shapes
+pytestmark = [pytest.mark.data_dependent_shapes, pytest.mark.unvectorized]
 
 
 @given(hh.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes(min_side=1)))

array_api_tests/test_sorting_functions.py

@@ -1,6 +1,7 @@
 import cmath
 from typing import Set
 
+import pytest
 from hypothesis import given
 from hypothesis import strategies as st
 from hypothesis.control import assume
@@ -29,6 +30,7 @@ def assert_scalar_in_set(
 
 
 # TODO: Test with signed zeros and NaNs (and ignore them somehow)
+@pytest.mark.unvectorized
 @given(
     x=hh.arrays(
         dtype=xps.real_dtypes(),
@@ -88,6 +90,7 @@ def test_argsort(x, data):
                     )
 
 
+@pytest.mark.unvectorized
 # TODO: Test with signed zeros and NaNs (and ignore them somehow)
 @given(
     x=hh.arrays(

array_api_tests/test_special_cases.py

@@ -1210,6 +1210,7 @@ def parse_binary_case_block(case_block: str) -> List[BinaryCase]:
 assert len(iop_params) != 0
 
 
+@pytest.mark.unvectorized
 @pytest.mark.parametrize("func_name, func, case", unary_params)
 @given(
     x=hh.arrays(dtype=xps.floating_dtypes(), shape=hh.shapes(min_side=1)),
@@ -1248,6 +1249,7 @@ def test_unary(func_name, func, case, x, data):
 )
 
 
+@pytest.mark.unvectorized
 @pytest.mark.parametrize("func_name, func, case", binary_params)
 @given(x1=x1_strat, x2=x2_strat, data=st.data())
 def test_binary(func_name, func, case, x1, x2, data):
@@ -1292,6 +1294,7 @@ def test_binary(func_name, func, case, x1, x2, data):
     assume(good_example)
 
 
+@pytest.mark.unvectorized
 @pytest.mark.parametrize("iop_name, iop, case", iop_params)
 @given(
     oneway_dtypes=hh.oneway_promotable_dtypes(dh.real_float_dtypes),

array_api_tests/test_statistical_functions.py

@@ -23,6 +23,7 @@ def kwarg_dtypes(dtype: DataType) -> st.SearchStrategy[Optional[DataType]]:
     return st.none() | st.sampled_from(dtypes)
 
 
+@pytest.mark.unvectorized
 @given(
     x=hh.arrays(
         dtype=xps.real_dtypes(),
@@ -75,6 +76,7 @@ def test_mean(x, data):
     # Values testing mean is too finicky
 
 
+@pytest.mark.unvectorized
 @given(
     x=hh.arrays(
         dtype=xps.real_dtypes(),
@@ -105,6 +107,7 @@ def test_min(x, data):
         ph.assert_scalar_equals("min", type_=scalar_type, idx=out_idx, out=min_, expected=expected)
 
 
+@pytest.mark.unvectorized
 @given(
     x=hh.arrays(
         dtype=xps.numeric_dtypes(),
@@ -193,6 +196,7 @@ def test_std(x, data):
     # We can't easily test the result(s) as standard deviation methods vary a lot
 
 
+@pytest.mark.unvectorized
 @pytest.mark.skip("flaky")  # TODO: fix!
 @given(
     x=hh.arrays(
@@ -245,6 +249,7 @@ def test_sum(x, data):
         ph.assert_scalar_equals("sum", type_=scalar_type, idx=out_idx, out=sum_, expected=expected)
 
 
+@pytest.mark.unvectorized
 @pytest.mark.skip(reason="flaky")  # TODO: fix!
 @given(
     x=hh.arrays(

array_api_tests/test_utility_functions.py

@@ -1,3 +1,4 @@
+import pytest
 from hypothesis import given
 from hypothesis import strategies as st
 
@@ -9,6 +10,7 @@
 from . import xps
 
 
+@pytest.mark.unvectorized
 @given(
     x=hh.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes(min_side=1)),
     data=st.data(),
@@ -36,6 +38,7 @@ def test_all(x, data):
                                 out=result, expected=expected, kw=kw)
 
 
+@pytest.mark.unvectorized
 @given(
     x=hh.arrays(dtype=xps.scalar_dtypes(), shape=hh.shapes()),
     data=st.data(),