Tokenizer test (pytorch-labs#21)

Summary:
X-link: pytorch/executorch#8586


Test Plan:
## OSS 
Build
```
cmake .     -DCMAKE_INSTALL_PREFIX=cmake-out -DTOKENIZERS_BUILD_TEST=ON -Bcmake-out
cmake --build cmake-out -j9 --target install
```

Test
```
(executorch) [[email protected] /data/users/lfq/tokenizers/cmake-out (lfq.tokenizer-test)]$ ctest
Test project /data/users/lfq/tokenizers/cmake-out
    Start 1: test_base64
1/5 Test pytorch-labs#1: test_base64 ......................   Passed    0.00 sec
    Start 2: test_llama2c_tokenizer
2/5 Test pytorch-labs#2: test_llama2c_tokenizer ...........   Passed    0.00 sec
    Start 3: test_pre_tokenizer
3/5 Test pytorch-labs#3: test_pre_tokenizer ...............   Passed    0.73 sec
    Start 4: test_sentencepiece
4/5 Test pytorch-labs#4: test_sentencepiece ...............   Passed    0.04 sec
    Start 5: test_tiktoken
5/5 Test pytorch-labs#5: test_tiktoken ....................   Passed    3.32 sec

100% tests passed, 0 tests failed out of 5

Total Test time (real) =   4.10 sec
```

## Internal
```
 buck2 test fbsource//xplat/pytorch/tokenizers/test:
 buck2 test fbcode//pytorch/tokenizers/test:
```

Reviewed By: larryliu0820

Differential Revision: D69860352

Pulled By: lucylq

targets.bzl

@@ -15,6 +15,7 @@ def define_common_targets():
         ]),
         visibility = [
             "@EXECUTORCH_CLIENTS",
+            "//pytorch/tokenizers/...",
         ],
         header_namespace = "",
     )
@@ -29,12 +30,14 @@ def define_common_targets():
         ],
         visibility = [
             "@EXECUTORCH_CLIENTS",
+            "//pytorch/tokenizers/...",
         ],
         compiler_flags = [
             "-D_USE_INTERNAL_STRING_VIEW",
         ],
         external_deps = [
             "sentencepiece",
+            "abseil-cpp",
         ],
     )
 
@@ -49,6 +52,7 @@ def define_common_targets():
         ],
         visibility = [
             "@EXECUTORCH_CLIENTS",
+            "//pytorch/tokenizers/...",
         ],
         compiler_flags = [
             "-D_USE_INTERNAL_STRING_VIEW",
@@ -84,6 +88,7 @@ def define_common_targets():
         ],
         visibility = [
             "@EXECUTORCH_CLIENTS",
+            "//pytorch/tokenizers/...",
         ],
         compiler_flags = [
             "-D_USE_INTERNAL_STRING_VIEW",
@@ -104,5 +109,6 @@ def define_common_targets():
         ],
         visibility = [
             "@EXECUTORCH_CLIENTS",
+            "//pytorch/tokenizers/...",
         ],
     )

test/TARGETS

@@ -0,0 +1,8 @@
+# Any targets that should be shared between fbcode and xplat must be defined in
+# targets.bzl. This file can contain fbcode-only targets.
+
+load(":targets.bzl", "define_common_targets")
+
+oncall("executorch")
+
+define_common_targets()

test/targets.bzl

@@ -0,0 +1,87 @@
+load(
+    "@fbsource//tools/build_defs:default_platform_defs.bzl",
+    "ANDROID",
+    "CXX",
+)
+load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime")
+
+def define_common_targets():
+    """Defines targets that should be shared between fbcode and xplat.
+
+    The directory containing this targets.bzl file should also contain both
+    TARGETS and BUCK files that call this function.
+    """
+    runtime.cxx_test(
+        name = "test_base64",
+        srcs = [
+            "test_base64.cpp",
+        ],
+        deps = [
+            "//pytorch/tokenizers:headers",
+        ],
+    )
+
+    runtime.cxx_test(
+        name = "test_llama2c_tokenizer",
+        srcs = [
+            "test_llama2c_tokenizer.cpp",
+        ],
+        deps = [
+            "//pytorch/tokenizers:llama2c_tokenizer",
+        ],
+        env = {
+            "RESOURCES_PATH": "$(location :resources)/resources",
+        },
+        platforms = [CXX, ANDROID],  # Cannot bundle resources on Apple platform.
+    )
+
+    runtime.cxx_test(
+        name = "test_pre_tokenizer",
+        srcs = [
+            "test_pre_tokenizer.cpp",
+        ],
+        deps = [
+            "//pytorch/tokenizers:headers",
+            "//pytorch/tokenizers:hf_tokenizer",
+        ],
+    )
+
+    runtime.cxx_test(
+        name = "test_sentencepiece",
+        srcs = [
+            "test_sentencepiece.cpp",
+        ],
+        deps = ["//pytorch/tokenizers:sentencepiece"],
+        external_deps = [
+            "sentencepiece",
+            "abseil-cpp",
+        ],
+        env = {
+            "RESOURCES_PATH": "$(location :resources)/resources",
+        },
+        platforms = [CXX, ANDROID],  # Cannot bundle resources on Apple platform.
+    )
+
+    runtime.cxx_test(
+        name = "test_tiktoken",
+        srcs = [
+            "test_tiktoken.cpp",
+        ],
+        deps = [
+            "//pytorch/tokenizers:tiktoken",
+        ],
+        env = {
+            "RESOURCES_PATH": "$(location :resources)/resources",
+        },
+        platforms = [CXX, ANDROID],  # Cannot bundle resources on Apple platform.
+        external_deps = [
+            "re2",
+        ],
+    )
+
+    runtime.filegroup(
+        name = "resources",
+        srcs = native.glob([
+            "resources/**",
+        ]),
+    )

test/test_llama2c_tokenizer.cpp

@@ -1,10 +1,4 @@
-/*
- * Copyright (c) Meta Platforms, Inc. and affiliates.
- * All rights reserved.
- *
- * This source code is licensed under the BSD-style license found in the
- * LICENSE file in the root directory of this source tree.
- */
+// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
 
 #ifdef TOKENIZERS_FB_BUCK
 #include <TestResourceUtils/TestResourceUtils.h>

test/test_sentencepiece.cpp

@@ -7,24 +7,15 @@
  */
 // @lint-ignore-every LICENSELINT
 
-#ifdef TOKENIZERS_FB_BUCK
-#include <TestResourceUtils/TestResourceUtils.h>
-#endif
 #include <gtest/gtest.h>
 #include <pytorch/tokenizers/sentencepiece.h>
 
 namespace tokenizers {
 
 namespace {
 static inline std::string _get_resource_path(const std::string& name) {
-#ifdef TOKENIZERS_FB_BUCK
-  return facebook::xplat::testing::getPathForTestResource(
-      "test/resources/" + name);
-#else
   return std::getenv("RESOURCES_PATH") + std::string("/") + name;
-#endif
 }
-
 } // namespace
 
 TEST(SPTokenizerTest, TestEncodeWithoutLoad) {

test/test_tiktoken.cpp

@@ -7,9 +7,6 @@
  */
 // @lint-ignore-every LICENSELINT
 
-#ifdef TOKENIZERS_FB_BUCK
-#include <TestResourceUtils/TestResourceUtils.h>
-#endif
 #include <gtest/gtest.h>
 #include <pytorch/tokenizers/tiktoken.h>
 
@@ -45,12 +42,7 @@ static inline std::unique_ptr<std::vector<std::string>> _get_special_tokens() {
 }
 
 static inline std::string _get_resource_path(const std::string& name) {
-#ifdef TOKENIZERS_FB_BUCK
-  return facebook::xplat::testing::getPathForTestResource(
-      "test/resources/" + name);
-#else
   return std::getenv("RESOURCES_PATH") + std::string("/") + name;
-#endif
 }
 
 } // namespace