Use simple_template_for_pretrain

Author: yuiseki

recipes/A5000_24GB_x8/i18n-ml-wikipedia.yaml

@@ -4,8 +4,7 @@ model_name: tinyllama-ml-wikipedia-1.5T-v0.1
 output_base_dir: /data/output
 dataset_id: wikimedia/wikipedia
 dataset_load_config: 20231101.ml
-dataset_input_field_name: title
-dataset_output_field_name: text
+dataset_input_field_name: text
 dataset_train_split_seed: 42
 dataset_train_split_test_size: 0.2
 lora_r: 8

recipes/A5000_24GB_x8/i18n-ms-wikipedia.yaml

@@ -4,8 +4,7 @@ model_name: tinyllama-ms-wikipedia-1.5T-v0.1
 output_base_dir: /data/output
 dataset_id: wikimedia/wikipedia
 dataset_load_config: 20231101.ms
-dataset_input_field_name: title
-dataset_output_field_name: text
+dataset_input_field_name: text
 dataset_train_split_seed: 42
 dataset_train_split_test_size: 0.2
 lora_r: 8

recipes/A5000_24GB_x8/i18n-rm-wikipedia.yaml

@@ -4,8 +4,7 @@ model_name: tinyllama-rm-wikipedia-1.5T-v0.1
 output_base_dir: /data/output
 dataset_id: wikimedia/wikipedia
 dataset_load_config: 20231101.rm
-dataset_input_field_name: title
-dataset_output_field_name: text
+dataset_input_field_name: text
 dataset_train_split_seed: 42
 dataset_train_split_test_size: 0.2
 lora_r: 8

recipes/A5000_24GB_x8/i18n-si-wikipedia.yaml

@@ -4,8 +4,7 @@ model_name: tinyllama-si-wikipedia-1.5T-v0.1
 output_base_dir: /data/output
 dataset_id: wikimedia/wikipedia
 dataset_load_config: 20231101.si
-dataset_input_field_name: title
-dataset_output_field_name: text
+dataset_input_field_name: text
 dataset_train_split_seed: 42
 dataset_train_split_test_size: 0.2
 lora_r: 8

recipes/A5000_24GB_x8/i18n-sq-wikipedia.yaml

@@ -4,8 +4,7 @@ model_name: tinyllama-sq-wikipedia-1.5T-v0.1
 output_base_dir: /data/output
 dataset_id: wikimedia/wikipedia
 dataset_load_config: 20231101.sq
-dataset_input_field_name: title
-dataset_output_field_name: text
+dataset_input_field_name: text
 dataset_train_split_seed: 42
 dataset_train_split_test_size: 0.2
 lora_r: 8

recipes/A5000_24GB_x8/i18n-sr-wikipedia.yaml

@@ -4,8 +4,7 @@ model_name: tinyllama-sr-wikipedia-1.5T-v0.1
 output_base_dir: /data/output
 dataset_id: wikimedia/wikipedia
 dataset_load_config: 20231101.sr
-dataset_input_field_name: title
-dataset_output_field_name: text
+dataset_input_field_name: text
 dataset_train_split_seed: 42
 dataset_train_split_test_size: 0.2
 lora_r: 8

recipes/A5000_24GB_x8/i18n-ta-wikipedia.yaml

@@ -4,8 +4,7 @@ model_name: tinyllama-ta-wikipedia-1.5T-v0.1
 output_base_dir: /data/output
 dataset_id: wikimedia/wikipedia
 dataset_load_config: 20231101.ta
-dataset_input_field_name: title
-dataset_output_field_name: text
+dataset_input_field_name: text
 dataset_train_split_seed: 42
 dataset_train_split_test_size: 0.2
 lora_r: 8

recipes/A5000_24GB_x8/i18n-yo-wikipedia.yaml

@@ -4,8 +4,7 @@ model_name: tinyllama-yo-wikipedia-1.5T-v0.1
 output_base_dir: /data/output
 dataset_id: wikimedia/wikipedia
 dataset_load_config: 20231101.yo
-dataset_input_field_name: title
-dataset_output_field_name: text
+dataset_input_field_name: text
 dataset_train_split_seed: 42
 dataset_train_split_test_size: 0.2
 lora_r: 8

src/train.py

@@ -35,9 +35,7 @@ def load_yaml(file_path):
 def simple_template_for_pretrain(input) -> str:
     # inputから、2つ以上連続する改行を除去する
     input = "\n".join([line for line in input.splitlines() if line.strip() != ""])
-    template = f"""\
-    {input}\
-    """
+    template = input
     # Remove any leading whitespace characters from each line in the template.
     template = "\n".join([line.lstrip() for line in template.splitlines()])
     return template
@@ -127,9 +125,7 @@ def prepare_train_data(dataset_id):
         output_field_name = train_config["dataset_output_field_name"]
         if "dataset_output_field_values_to_texts" in train_config:
             output_field_values_to_texts = train_config["dataset_output_field_values_to_texts"]
-            data_df[output_field_name] = data_df[output_field_name].apply(
-                lambda x: output_field_values_to_texts.get(x, x)
-            )
+            data_df[output_field_name] = data_df[output_field_name].apply(lambda x: output_field_values_to_texts.get(x, x))
         if "dataset_context_field_name" in train_config:
             context_field_name = train_config["dataset_context_field_name"]
             if "dataset_context_hint" not in train_config: