max_seq_length=1024だと、イタリア語Wikipediaが読み込めない

Author: yuiseki

recipes/A5000_24GB_x8/Mistral-7B-v0.1/i18n-fr-wikipedia.yaml

@@ -0,0 +1,16 @@
+target_task: tasks/i18n/ja.md
+base_model_id: mistralai/Mistral-7B-v0.1
+model_name: Mistral-7B-v0.1-fr-wikipedia-v0.1
+output_base_dir: output
+dataset_id: wikimedia/wikipedia
+dataset_load_config: 20231101.fr
+dataset_input_field_name: text
+dataset_train_split_seed: 42
+dataset_train_split_test_size: 0.2
+lora_r: 8
+lora_alpha: 16
+lora_dropout: 0.05
+train_claim_gpu_num: 8
+train_per_device_train_batch_size: 1
+train_gradient_accumulation_steps: 16
+train_num_train_epochs: 2

recipes/A5000_24GB_x8/Mistral-7B-v0.1/i18n-it-wikipedia.yaml

@@ -0,0 +1,16 @@
+target_task: tasks/i18n/ja.md
+base_model_id: mistralai/Mistral-7B-v0.1
+model_name: Mistral-7B-v0.1-it-wikipedia-v0.1
+output_base_dir: output
+dataset_id: wikimedia/wikipedia
+dataset_load_config: 20231101.it
+dataset_input_field_name: text
+dataset_train_split_seed: 42
+dataset_train_split_test_size: 0.2
+lora_r: 8
+lora_alpha: 16
+lora_dropout: 0.05
+train_claim_gpu_num: 8
+train_per_device_train_batch_size: 1
+train_gradient_accumulation_steps: 16
+train_num_train_epochs: 2

recipes/A5000_24GB_x8/Mistral-7B-v0.1/i18n-ja-wikipedia.yaml

@@ -13,4 +13,4 @@ lora_dropout: 0.05
 train_claim_gpu_num: 8
 train_per_device_train_batch_size: 1
 train_gradient_accumulation_steps: 16
-train_num_train_epochs: 2
+train_num_train_epochs: 2

src/train.py

@@ -281,7 +281,7 @@ def load_model_and_tokenizer(model_id):
     args=training_arguments,
     tokenizer=tokenizer,
     packing=False,
-    max_seq_length=1024,
+    max_seq_length=512,
 )
 
 #