From 2bc43b16bd89bb0c8f513db41a3d251dd6ad7348 Mon Sep 17 00:00:00 2001 From: lucylq Date: Wed, 19 Feb 2025 14:13:30 -0800 Subject: [PATCH] Tokenizer test (#21) Summary: Pull Request resolved: https://github.com/pytorch-labs/tokenizers/pull/21 Test Plan: ## OSS Build ``` cmake . -DCMAKE_INSTALL_PREFIX=cmake-out -DTOKENIZERS_BUILD_TEST=ON -Bcmake-out cmake --build cmake-out -j9 --target install ``` Test ``` (executorch) [lfq@devvm20128.prn0 /data/users/lfq/tokenizers/cmake-out (lfq.tokenizer-test)]$ ctest Test project /data/users/lfq/tokenizers/cmake-out Start 1: test_base64 1/5 Test https://github.com/pytorch-labs/tokenizers/issues/1: test_base64 ...................... Passed 0.00 sec Start 2: test_llama2c_tokenizer 2/5 Test https://github.com/pytorch-labs/tokenizers/issues/2: test_llama2c_tokenizer ........... Passed 0.00 sec Start 3: test_pre_tokenizer 3/5 Test https://github.com/pytorch-labs/tokenizers/issues/3: test_pre_tokenizer ............... Passed 0.73 sec Start 4: test_sentencepiece 4/5 Test https://github.com/pytorch-labs/tokenizers/issues/4: test_sentencepiece ............... Passed 0.04 sec Start 5: test_tiktoken 5/5 Test https://github.com/pytorch-labs/tokenizers/issues/5: test_tiktoken .................... Passed 3.32 sec 100% tests passed, 0 tests failed out of 5 Total Test time (real) = 4.10 sec ``` ## Internal ``` buck2 test fbsource//xplat/pytorch/tokenizers/test: buck2 test fbcode//pytorch/tokenizers/test: ``` Differential Revision: D69860352 Pulled By: lucylq --- shim/xplat/executorch/build/runtime_wrapper.bzl | 2 +- test/utils/targets.bzl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/shim/xplat/executorch/build/runtime_wrapper.bzl b/shim/xplat/executorch/build/runtime_wrapper.bzl index 03bca6623f9..b81aabcd83f 100644 --- a/shim/xplat/executorch/build/runtime_wrapper.bzl +++ b/shim/xplat/executorch/build/runtime_wrapper.bzl @@ -171,7 +171,7 @@ def _patch_kwargs_common(kwargs): # don't pick up unexpected clients while things are still in flux. if not kwargs.pop("_is_external_target", False): for target in kwargs.get("visibility", []): - if not (target.startswith("//executorch") or target.startswith("@")): + if not (target.startswith("//executorch") or target.startswith("//pytorch/tokenizers") or target.startswith("@")): fail("Please manage all external visibility using the " + "EXECUTORCH_CLIENTS list in " + "//executorch/build/fb/clients.bzl. " + diff --git a/test/utils/targets.bzl b/test/utils/targets.bzl index b16ce2bac25..93e33daf81f 100644 --- a/test/utils/targets.bzl +++ b/test/utils/targets.bzl @@ -21,6 +21,7 @@ def define_common_targets(): ], visibility = [ "//executorch/...", + "//pytorch/tokenizers/...", "@EXECUTORCH_CLIENTS", ], deps = [