fix default output features

Signed-off-by: ashors1 <[email protected]>
google-research · Dec 19, 2024 · 9cc0d6f · 9cc0d6f
1 parent 5f03619
commit 9cc0d6f
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 6 deletions.
diff --git a/t5x/contrib/gpu/scripts_gpu/dummy_wikipedia_config/dummy_wikipedia_seqio.py b/t5x/contrib/gpu/scripts_gpu/dummy_wikipedia_config/dummy_wikipedia_seqio.py
@@ -22,10 +22,16 @@
 
 DEFAULT_OUTPUT_FEATURES = {
     "inputs": seqio.Feature(
-        vocabulary=t5.data.get_default_vocabulary(), add_eos=True,
+        vocabulary=seqio.SentencePieceVocabulary(
+            sentencepiece_model_file="gs://t5-data/vocabs/cc_all.32000.100extra/sentencepiece.model",
+        ),
+        add_eos=True,
         required=False),
     "targets": seqio.Feature(
-        vocabulary=t5.data.get_default_vocabulary(), add_eos=True)
+        vocabulary=seqio.SentencePieceVocabulary(
+            sentencepiece_model_file="gs://t5-data/vocabs/cc_all.32000.100extra/sentencepiece.model",
+        ),
+        add_eos=True)
 }
 
 # ================================ Wikipedia ===================================

diff --git a/t5x/contrib/gpu/scripts_gpu/seqio_tasks.py b/t5x/contrib/gpu/scripts_gpu/seqio_tasks.py
@@ -34,11 +34,20 @@
 
 DEFAULT_OUTPUT_FEATURES = {
     "inputs":
-        seqio.Feature(vocabulary=t5.data.get_default_vocabulary(),
-                      add_eos=True,
-                      required=False),
+        seqio.Feature(
+            vocabulary=seqio.SentencePieceVocabulary(
+                sentencepiece_model_file="gs://t5-data/vocabs/cc_all.32000.100extra/sentencepiece.model",
+            ),
+            add_eos=True,
+            required=False
+        ),
     "targets":
-        seqio.Feature(vocabulary=t5.data.get_default_vocabulary(), add_eos=True)
+        seqio.Feature(
+            vocabulary=seqio.SentencePieceVocabulary(
+                sentencepiece_model_file="gs://t5-data/vocabs/cc_all.32000.100extra/sentencepiece.model",
+            ),
+            add_eos=True
+        )
 }
 
 # ================================== The Pile ====================================