Add tokenizer trainer script to create and save a WordLevel tokenizer

Author: alifa98

tokenizer_trainer.py

@@ -0,0 +1,24 @@
+from tokenizers import Tokenizer
+from tokenizers.models import WordLevel
+from tokenizers.trainers import WordLevelTrainer
+from tokenizers.pre_tokenizers import Whitespace
+from transformers import PreTrainedTokenizerFast
+
+# Initialize the tokenizer
+tokenizer = Tokenizer(WordLevel(unk_token="[UNK]"))
+tokenizer.pre_tokenizer = Whitespace()
+
+# Trainer to learn the vocabulary
+trainer = WordLevelTrainer(special_tokens=["[UNK]", "[PAD]", "[CLS]", "[SEP]"])
+tokenizer.train(files=["sequences.txt"], trainer=trainer)
+
+hf_tokenizer = PreTrainedTokenizerFast(
+    tokenizer_object=tokenizer,
+    unk_token="[UNK]",
+    pad_token="[PAD]",
+    cls_token="[CLS]",
+    sep_token="[SEP]"
+)
+
+# Save the tokenizer for future use
+hf_tokenizer.save_pretrained("./ho-sequence-tokenizer")