Try testing more pytest (#14037)

This adds 35-40 new tests (because we weren't running before).

No additional net new ignores

Allow based on top level directories that way new tests are enabled
automatically.

Explicitly ignore individual failing tests.

Author: mergennachin

pytest.ini

@@ -8,90 +8,109 @@ addopts =
     --capture=sys
     # don't suppress warnings, but don't shove them all to the end either
     -p no:warnings
-    # Ignore backends/arm tests you need to run examples/arm/setup.sh to install some tool to make them work
-    # For GitHub testing this is setup/executed in the unittest-arm job see .github/workflows/pull.yml for more info.
-    --ignore-glob=backends/arm/**/*
-    # explicitly list out tests that are running successfully in oss
+    
+    # === TEST DIRECTORIES TO RUN ===
+    
+    # ci/scripts
     .ci/scripts/tests
-    examples/models/test
+    
+    # backends
+    backends/apple/coreml/test
+    backends/test/harness/tests
+    backends/test/suite/tests
+    backends/transforms
+    backends/xnnpack/test
+    # Ignore ARM backend - requires running examples/arm/setup.sh to install tools
+    --ignore-glob=backends/arm/**/*
+    # Ignore failing xnnpack tests
+    --ignore=backends/xnnpack/test/ops/test_linear.py
+    --ignore=backends/xnnpack/test/ops/test_bmm.py
+    --ignore=backends/xnnpack/test/quantizer/test_xnnpack_quantizer.py
+    # Ignore backends/test root - WIP testing infra, see https://github.com/pytorch/executorch/discussions/11140
+    --ignore=backends/test
+    
+    # codegen
+    codegen/test
+    
+    # devtools
     devtools/
+    # Ignore test with missing dependencies
     --ignore=devtools/visualization/visualization_utils_test.py
+    
     # examples
+    examples/models/test
     examples/models/llama/tests
     examples/models/llama/config
     examples/models/llama3_2_vision/preprocess
     examples/models/llama3_2_vision/vision_encoder/test
     examples/models/llama3_2_vision/text_decoder/test
-    # examples/models/llava/test TODO: enable this
+    examples/models/llava/test
+    # Ignore test requiring torchao - T200992559: Add torchao to ET as core dependency
+    --ignore=examples/models/llama/tests/test_pre_quantization_transforms.py
+    # Failing test
+    --ignore=examples/models/llama/tests/test_ring_attention.py
+    # Ignore test with missing dependencies
+    --ignore=examples/models/llava/test/test_pte.py
+    # Ignore failing llava tests (missing accelerate dependency)
+    --ignore=examples/models/llava/test/test_llava.py
+    
     # exir
-    exir/_serialize/test
-    exir/backend/test
-    exir/dialects/backend/test
-    exir/dialects/edge/test
-    exir/dialects/test
-    exir/emit/test
-    exir/program/test
-    exir/tests/
-    # executorch/export
+    exir/
+    # Ignore tests with missing custom_ops_generated_lib dependencies
+    --ignore=exir/backend/test/demos/test_delegate_aten_mode.py
+    --ignore=exir/backend/test/demos/test_xnnpack_qnnpack.py
+    --ignore=exir/tests/test_memory_format_ops_pass_aten.py
+    --ignore=exir/tests/test_memory_planning.py
+    --ignore=exir/tests/test_passes.py
+    --ignore=exir/tests/test_quantization.py
+    --ignore=exir/tests/test_verification.py
+    --ignore=exir/verification/test/test_verifier.py
+    # Ignore failing tests
+    --ignore=exir/backend/test/demos/rpc/test_rpc.py
+    --ignore=exir/backend/test/test_backends.py
+    --ignore=exir/backend/test/test_backends_lifted.py
+    --ignore=exir/backend/test/test_partitioner.py
+    --ignore=exir/operator/test/test_operator.py
+    --ignore=exir/tests/test_common.py
+    --ignore=exir/tests/test_op_convert.py
+    
+    # export
     export/tests
     --ignore=export/tests/test_export_stages.py
-    # kernels/
+    
+    # extension
+    extension/
+    # Ignore tests with missing dependencies or build issues
+    --ignore=extension/flat_tensor/test/test_serialize.py
+    --ignore=extension/llm/custom_ops/test_preprocess_custom_ops.py
+    --ignore=extension/llm/tokenizers/third-party/re2/python/re2_test.py
+    --ignore=extension/llm/tokenizers/third-party/sentencepiece/python/test/sentencepiece_test.py
+    # Ignore failing tokenizer tests
+    --ignore=extension/llm/tokenizers/test/test_tekken_python.py
+    
+    # kernels
     kernels/prim_ops/test
     kernels/quantized
-    # Because this test depends on test only cpp ops lib
-    # Will add test only cmake targets to re-enable this test
-    # but maybe it is a bit of anti-pattern
-    --ignore=kernels/quantized/test/test_quant_dequant_per_token.py
     kernels/test/test_case_gen.py
-    # backends/test
-    # This effort is WIP and will be enabled in CI once testing infra
-    # is stable and signal to noise ratio is good (no irrelevant failures).
-    # See https://github.com/pytorch/executorch/discussions/11140
-    --ignore=backends/test
-    backends/test/harness/tests
-    backends/test/suite/tests
-    # backends/xnnpack
-    backends/xnnpack/test/ops
-    --ignore=backends/xnnpack/test/ops/test_bmm.py
-    --ignore=backends/xnnpack/test/ops/test_conv2d.py
-    --ignore=backends/xnnpack/test/ops/test_linear.py
-    --ignore=backends/xnnpack/test/ops/test_sdpa.py
-    backends/xnnpack/test/passes
-    backends/xnnpack/test/recipes
-    backends/xnnpack/test/serialization
-    # backends/apple/coreml
-    backends/apple/coreml/test
-    # extension/
-    extension/llm/modules/test
-    extension/llm/export
-    extension/llm/custom_ops/test_sdpa_with_kv_cache.py
-    extension/llm/custom_ops/test_update_cache.py
-    extension/llm/custom_ops/test_quantized_sdpa.py
-    extension/pybindings/test
-    extension/training/pybindings/test
-    # Runtime
+    # Ignore test depending on test-only cpp ops lib
+    --ignore=kernels/quantized/test/test_quant_dequant_per_token.py
+    
+    # profiler
+    profiler/
+    # Ignore test with missing dependencies
+    --ignore=profiler/test/test_profiler_e2e.py
+    
+    # runtime
     runtime
-    # Tools
-    codegen/test
+    
+    # test
+    test/
+    # Ignore tests with missing dependencies
+    --ignore=test/end2end/test_end2end.py
+    --ignore=test/end2end/test_temp_allocator_fix.py
+    
+    # tools
     tools/cmake
-    # test TODO: fix these tests
-    # test/end2end/test_end2end.py
-    --ignore=backends/xnnpack/test/ops/linear.py
-    --ignore=backends/xnnpack/test/models/llama2_et_example.py
-    # T200992559: Add torchao to ET as core dependency
-    --ignore=examples/models/llama/tests/test_pre_quantization_transforms.py
-    --ignore=exir/backend/test/demos
-    --ignore=exir/backend/test/test_backends.py
-    --ignore=exir/backend/test/test_backends_lifted.py
-    --ignore=exir/backend/test/test_partitioner.py
-    --ignore=exir/tests/test_common.py
-    --ignore=exir/tests/test_memory_format_ops_pass_aten.py
-    --ignore=exir/tests/test_memory_planning.py
-    --ignore=exir/tests/test_op_convert.py
-    --ignore=exir/tests/test_passes.py
-    --ignore=exir/tests/test_quant_fusion_pass.py
-    --ignore=exir/tests/test_quantization.py
-    --ignore=exir/tests/test_verification.py
 
 # run the same tests multiple times to determine their
 # flakiness status. Default to 50 re-runs