diff --git a/.gitignore b/.gitignore index 68bc17f..5d242b1 100644 --- a/.gitignore +++ b/.gitignore @@ -158,3 +158,5 @@ cython_debug/ # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ + +debug.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..5573147 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +repos: +- repo: https://github.com/pycqa/isort + rev: 5.12.0 + hooks: + - id: isort + name: isort (python) + args: ["--profile", "black", "--filter-files"] +- repo: https://github.com/psf/black + rev: 22.12.0 + hooks: + - id: black +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..bb23c28 --- /dev/null +++ b/Makefile @@ -0,0 +1,30 @@ +all: format clean pre test + echo 'finished' + +.PHONY: format +format: + isort --profile black --filter-files . + black . + +.PHONY: test +test: + coverage run --source rex -m pytest -vv . + coverage report -m + flake8 + +.PHONY: pre +pre: + pre-commit run --all-files + +.PHONY: debug +debug: + pytest -vv tests/tasks/test_re.py + +.PHONY: clean +clean: + rm -rf build/ + rm -rf dist/ + rm -rf *.egg-info/ + rm -f .coverage + rm -f coverage.xml + find . | grep -E '(__pycache__|\.pyc|\.pyo$$)' | xargs rm -rf diff --git a/README.md b/README.md index a72fc73..6e65239 100644 --- a/README.md +++ b/README.md @@ -1 +1,11 @@ -# smoe \ No newline at end of file +# smoe + +## For developers + +- Make sure the Python version `>=3.10` (a strict version contraint for better type hinting) + +```bash +$ pip install -e .[dev] +``` + + diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..bd52db8 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.0.0 \ No newline at end of file diff --git a/conf/deepspeed/bf16.json b/conf/deepspeed/bf16.json new file mode 100644 index 0000000..3c41ddb --- /dev/null +++ b/conf/deepspeed/bf16.json @@ -0,0 +1,20 @@ +{ + "bf16": { + "enabled": true + }, + "zero_optimization": { + "stage": 2, + "allgather_partitions": true, + "allgather_bucket_size": 1e8, + "overlap_comm": true, + "reduce_scatter": true, + "reduce_bucket_size": 1e8, + "contiguous_gradients": true + }, + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false +} \ No newline at end of file diff --git a/conf/deepspeed/fp16.json b/conf/deepspeed/fp16.json new file mode 100644 index 0000000..8e721f0 --- /dev/null +++ b/conf/deepspeed/fp16.json @@ -0,0 +1,26 @@ +{ + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 100, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1e-10 + }, + "zero_optimization": { + "stage": 2, + "allgather_partitions": true, + "allgather_bucket_size": 1e8, + "overlap_comm": true, + "reduce_scatter": true, + "reduce_bucket_size": 1e8, + "contiguous_gradients": true + }, + + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6c08761 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +scikit-learn>=1.3.0 +omegaconf>=2.0.6 +tqdm>=4.65.0 +datasets>=2.13.1 +transformers>=4.30.2 +peft>=0.4.0 \ No newline at end of file diff --git a/scripts/cpt/README.md b/scripts/cpt/README.md new file mode 100644 index 0000000..47e75f0 --- /dev/null +++ b/scripts/cpt/README.md @@ -0,0 +1,4 @@ +# Scripts for Continual Pre-training + +- `lora.sh`: Parameter-efficient tuning +- `fpt.sh`: Full-parameter pretraining diff --git a/scripts/cpt/fpt.sh b/scripts/cpt/fpt.sh new file mode 100644 index 0000000..3813859 --- /dev/null +++ b/scripts/cpt/fpt.sh @@ -0,0 +1,74 @@ +#!/usr/bin/bash + +#SBATCH --job-name=cpt-bf16-2nodes-woLora +#SBATCH --partition=MoE +#SBATCH --output=logs/%x.log +#SBATCH --error=logs/%x.log + +#SBATCH --nodes=2 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=8 + +source ~/anaconda3/bin/activate torch + +lr=2e-4 + +pretrained_model=/mnt/petrelfs/share_data/quxiaoye/models/llama_7B/ +tokenizer_path=/mnt/petrelfs/share_data/quxiaoye/models/llama_7B/ +dataset_dir=resources +data_cache=temp_data_cache_dir +per_device_train_batch_size=1 +per_device_eval_batch_size=1 +gradient_accumulation_steps=8 +output_dir=output_dir_cpt_ymcui + +deepspeed_config_file=conf/ds_bf16.json + +nodes=( $( scontrol show hostnames $SLURM_JOB_NODELIS ) ) +nodes_array=($nodes) +head_node=${nodes_array[0]} +head_node_ip=$(srun --nodes=1 --ntasks=1 -w "$head_node" hostname --ip-address) +echo "Node: $head_node" +echo "Node IP: $head_node_ip" +export LOGLEVEL=INFO + +srun torchrun \ + --nnodes 2 \ + --nproc_per_node 8 \ + --node_rank $SLURM_NODEID \ + --rdzv_id $RANDOM \ + --rdzv_backend c10d \ + --rdzv_endpoint $head_node:29518 \ + src/entrypoint/run_clm_pt_wo_peft.py \ + --deepspeed ${deepspeed_config_file} \ + --model_name_or_path ${pretrained_model} \ + --tokenizer_name_or_path ${tokenizer_path} \ + --dataset_dir ${dataset_dir} \ + --data_cache_dir ${data_cache} \ + --validation_split_percentage 0.001 \ + --per_device_train_batch_size ${per_device_train_batch_size} \ + --per_device_eval_batch_size ${per_device_eval_batch_size} \ + --do_train \ + --seed $RANDOM \ + --bf16 \ + --num_train_epochs 1 \ + --lr_scheduler_type cosine \ + --learning_rate ${lr} \ + --warmup_ratio 0.05 \ + --weight_decay 0.01 \ + --logging_strategy steps \ + --logging_steps 10 \ + --save_strategy steps \ + --save_total_limit 3 \ + --save_steps 200 \ + --gradient_accumulation_steps ${gradient_accumulation_steps} \ + --preprocessing_num_workers 8 \ + --block_size 512 \ + --output_dir ${output_dir} \ + --overwrite_output_dir \ + --ddp_timeout 30000 \ + --logging_first_step True \ + --torch_dtype bfloat16 \ + --gradient_checkpointing \ + --ddp_find_unused_parameters False diff --git a/scripts/cpt/lora.sh b/scripts/cpt/lora.sh new file mode 100644 index 0000000..fd5f6dd --- /dev/null +++ b/scripts/cpt/lora.sh @@ -0,0 +1,84 @@ +#!/usr/bin/bash + +#SBATCH --job-name=cpt-lora-bf16-2nodes +#SBATCH --partition=MoE +#SBATCH --output=logs/%x.log +#SBATCH --error=logs/%x.log + +#SBATCH --nodes=2 +#SBATCH --ntasks-per-node=1 +#SBATCH --gres=gpu:8 +#SBATCH --cpus-per-task=8 + +source ~/anaconda3/bin/activate torch + +lr=2e-4 +lora_rank=8 +lora_alpha=32 +lora_trainable="q_proj,v_proj,k_proj,o_proj,gate_proj,down_proj,up_proj" +modules_to_save="embed_tokens,lm_head" +lora_dropout=0.05 + +pretrained_model=/mnt/petrelfs/share_data/quxiaoye/models/llama_7B/ +tokenizer_path=/mnt/petrelfs/share_data/quxiaoye/models/llama_7B/ +dataset_dir=resources +data_cache=temp_data_cache_dir +per_device_train_batch_size=1 +per_device_eval_batch_size=1 +gradient_accumulation_steps=8 +output_dir=output_dir + +deepspeed_config_file=conf/ds_bf16.json + +nodes=( $( scontrol show hostnames $SLURM_JOB_NODELIS ) ) +nodes_array=($nodes) +head_node=${nodes_array[0]} +head_node_ip=$(srun --nodes=1 --ntasks=1 -w "$head_node" hostname --ip-address) +echo "Node: $head_node" +echo "Node IP: $head_node_ip" +export LOGLEVEL=INFO + +srun torchrun \ + --nnodes 2 \ + --nproc_per_node 8 \ + --node_rank $SLURM_NODEID \ + --rdzv_id $RANDOM \ + --rdzv_backend c10d \ + --rdzv_endpoint $head_node:29518 \ + src/entrypoint/run_clm_pt_with_peft.py \ + --deepspeed ${deepspeed_config_file} \ + --model_name_or_path ${pretrained_model} \ + --tokenizer_name_or_path ${tokenizer_path} \ + --dataset_dir ${dataset_dir} \ + --data_cache_dir ${data_cache} \ + --validation_split_percentage 0.001 \ + --per_device_train_batch_size ${per_device_train_batch_size} \ + --per_device_eval_batch_size ${per_device_eval_batch_size} \ + --do_train \ + --seed $RANDOM \ + --bf16 \ + --num_train_epochs 1 \ + --lr_scheduler_type cosine \ + --learning_rate ${lr} \ + --warmup_ratio 0.05 \ + --weight_decay 0.01 \ + --logging_strategy steps \ + --logging_steps 10 \ + --save_strategy steps \ + --save_total_limit 3 \ + --save_steps 200 \ + --gradient_accumulation_steps ${gradient_accumulation_steps} \ + --preprocessing_num_workers 8 \ + --block_size 512 \ + --output_dir ${output_dir} \ + --overwrite_output_dir \ + --ddp_timeout 30000 \ + --logging_first_step True \ + --lora_rank ${lora_rank} \ + --lora_alpha ${lora_alpha} \ + --trainable ${lora_trainable} \ + --modules_to_save ${modules_to_save} \ + --lora_dropout ${lora_dropout} \ + --torch_dtype float16 \ + --gradient_checkpointing \ + --ddp_find_unused_parameters False diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..2548563 --- /dev/null +++ b/setup.py @@ -0,0 +1,49 @@ +import os + +import setuptools + +readme_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "README.md") +with open(readme_filepath, "r") as fh: + long_description = fh.read() + +version_filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), "VERSION") +with open(version_filepath, "r") as fh: + version = fh.read().strip() + +setuptools.setup( + name="smoe", + version=version, + author="MoE Group", + author_email="tzhu1997@outlook.com", + description="A toolkit for LLM MoE and continual pretraining.", + long_description_content_type="text/markdown", + long_description=long_description, + url="https://github.com/Spico197/smoe", + packages=setuptools.find_packages(exclude=["tests", "tests.*", "docs", "docs.*"]), + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + ], + python_requires=">=3.10", + install_requires=[ + "scikit-learn>=1.3.0", + "omegaconf>=2.0.6", + "tqdm>=4.65.0", + "datasets>=2.13.1", + "transformers>=4.30.2", + "peft>=0.4.0", + ], + extras_require={ + "dev": [ + "pytest", + "coverage", + "black", + "isort", + "flake8", + "pre-commit", + ] + }, + include_package_data=True, + entry_points={}, +) diff --git a/smoe/__init__.py b/smoe/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/smoe/callbacks/__init__.py b/smoe/callbacks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/smoe/callbacks/save_peft_model.py b/smoe/callbacks/save_peft_model.py new file mode 100644 index 0000000..8164397 --- /dev/null +++ b/smoe/callbacks/save_peft_model.py @@ -0,0 +1,32 @@ +import os + +from transformers import TrainerCallback +from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR + + +class SavePeftModelCallback(TrainerCallback): + def __init__(self, peft_model_subdir: str = "peft_model"): + self.peft_model_subdir = peft_model_subdir + + def save_model(self, args, state, **kwargs): + if state.best_model_checkpoint is not None: + checkpoint_folder = os.path.join( + state.best_model_checkpoint, self.peft_model_subdir + ) + else: + checkpoint_folder = os.path.join( + args.output_dir, f"{PREFIX_CHECKPOINT_DIR}-{state.global_step}" + ) + + peft_model_path = os.path.join(checkpoint_folder, self.peft_model_subdir) + kwargs["model"].save_pretrained(peft_model_path) + kwargs["tokenizer"].save_pretrained(peft_model_path) + + def on_save(self, args, state, control, **kwargs): + self.save_model(args, state, **kwargs) + return control + + def on_train_end(self, args, state, control, **kwargs): + peft_model_path = os.path.join(args.output_dir, self.peft_model_subdir) + kwargs["model"].save_pretrained(peft_model_path) + kwargs["tokenizer"].save_pretrained(peft_model_path) diff --git a/smoe/data/__init__.py b/smoe/data/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/smoe/data/collate_fn.py b/smoe/data/collate_fn.py new file mode 100644 index 0000000..a83e0c7 --- /dev/null +++ b/smoe/data/collate_fn.py @@ -0,0 +1,65 @@ +from typing import Any, Mapping + +import torch +import numpy as np + + +def fault_tolerance_data_collator(features: list) -> dict[str, Any]: + if not isinstance(features[0], Mapping): + features = [vars(f) for f in features] + first = features[0] + batch = {} + + # Special handling for labels. + # Ensure that tensor is created with the correct type + # (it should be automatically the case, but let's make sure of it.) + if "label" in first and first["label"] is not None: + label = ( + first["label"].item() + if isinstance(first["label"], torch.Tensor) + else first["label"] + ) + dtype = torch.long if isinstance(label, int) else torch.float + batch["labels"] = torch.tensor([f["label"] for f in features], dtype=dtype) + elif "label_ids" in first and first["label_ids"] is not None: + if isinstance(first["label_ids"], torch.Tensor): + batch["labels"] = torch.stack([f["label_ids"] for f in features]) + else: + dtype = ( + torch.long if isinstance(first["label_ids"][0], int) else torch.float + ) + batch["labels"] = torch.tensor( + [f["label_ids"] for f in features], dtype=dtype + ) + + # Handling of all other possible keys. + # Again, we will use the first element to figure out which key/values are not None for this model. + + try: + for k, v in first.items(): + if ( + k not in ("label", "label_ids") + and v is not None + and not isinstance(v, str) + ): + if isinstance(v, torch.Tensor): + batch[k] = torch.stack([f[k] for f in features]) + elif isinstance(v, np.ndarray): + batch[k] = torch.tensor(np.stack([f[k] for f in features])) + else: + batch[k] = torch.tensor([f[k] for f in features]) + except ValueError: # quick fix by simply take the first example + for k, v in first.items(): + if ( + k not in ("label", "label_ids") + and v is not None + and not isinstance(v, str) + ): + if isinstance(v, torch.Tensor): + batch[k] = torch.stack([features[0][k]] * len(features)) + elif isinstance(v, np.ndarray): + batch[k] = torch.tensor(np.stack([features[0][k]] * len(features))) + else: + batch[k] = torch.tensor([features[0][k]] * len(features)) + + return batch diff --git a/smoe/entrypoint/__init__.py b/smoe/entrypoint/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/smoe/entrypoint/cpt_lora.py b/smoe/entrypoint/cpt_lora.py new file mode 100644 index 0000000..9111392 --- /dev/null +++ b/smoe/entrypoint/cpt_lora.py @@ -0,0 +1,385 @@ +import math +import os +import sys +from itertools import chain +from pathlib import Path +import datasets +import torch +from datasets import load_dataset, concatenate_datasets + +import transformers +from transformers import ( + CONFIG_MAPPING, + AutoConfig, + AutoModelForCausalLM, + LlamaForCausalLM, + LlamaTokenizer, + AutoTokenizer, + Trainer, + is_torch_tpu_available, + set_seed, +) +from transformers.testing_utils import CaptureLogger +from transformers.trainer_utils import get_last_checkpoint + +from peft import ( + LoraConfig, + TaskType, + get_peft_model, + PeftModel, + get_peft_model_state_dict, +) + +from smoe.callbacks.save_peft_model import SavePeftModelCallback +from smoe.metrics.preprocess import logits_argmax +from smoe.metrics.accuracy import compute_metrics +from smoe.data.collate_fn import fault_tolerance_data_collator +from smoe.utils.config import parse_args +from smoe.utils.config import ModelArguments, DataArguments, LoraTrainingArguments +from smoe.utils.logging import get_logger_from_training_args + + +def main(): + model_args, data_args, training_args = parse_args(ModelArguments, DataArguments, LoraTrainingArguments) + logger = get_logger_from_training_args(__name__, training_args) + logger.warning( + f"Process local rank: {training_args.local_rank}, " + f"device: {training_args.device}, " + f"n_gpu: {training_args.n_gpu}, " + f"distributed training: {bool(training_args.local_rank != -1)}, " + f"fp16 training: {training_args.fp16}, " + f"bf16 training: {training_args.bf16}" + ) + + # Detecting last checkpoint. + last_checkpoint = None + if ( + os.path.isdir(training_args.output_dir) + and training_args.do_train + and not training_args.overwrite_output_dir + ): + last_checkpoint = get_last_checkpoint(training_args.output_dir) + if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0: + raise ValueError( + f"Output directory ({training_args.output_dir}) already exists and is not empty. " + "Use --overwrite_output_dir to overcome." + ) + elif ( + last_checkpoint is not None and training_args.resume_from_checkpoint is None + ): + logger.info( + f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change " + "the `--output_dir` or add `--overwrite_output_dir` to train from scratch." + ) + + # Set seed before initializing model. + set_seed(training_args.seed) + + config_kwargs = { + "cache_dir": model_args.cache_dir, + "revision": model_args.model_revision, + "use_auth_token": True if model_args.use_auth_token else None, + } + if model_args.config_name: + config = AutoConfig.from_pretrained(model_args.config_name, **config_kwargs) + elif model_args.model_name_or_path: + config = AutoConfig.from_pretrained( + model_args.model_name_or_path, **config_kwargs + ) + else: + config = CONFIG_MAPPING[model_args.model_type]() + logger.warning("You are instantiating a new config instance from scratch.") + if model_args.config_overrides is not None: + logger.info(f"Overriding config: {model_args.config_overrides}") + config.update_from_string(model_args.config_overrides) + logger.info(f"New config: {config}") + + tokenizer_kwargs = { + "cache_dir": model_args.cache_dir, + "use_fast": model_args.use_fast_tokenizer, + "revision": model_args.model_revision, + "use_auth_token": True if model_args.use_auth_token else None, + } + if model_args.tokenizer_name: + tokenizer = AutoTokenizer.from_pretrained( + model_args.tokenizer_name, **tokenizer_kwargs + ) + elif model_args.tokenizer_name_or_path: + tokenizer = LlamaTokenizer.from_pretrained( + model_args.tokenizer_name_or_path, **tokenizer_kwargs + ) + else: + raise ValueError( + "You are instantiating a new tokenizer from scratch. This is not supported by this script." + "You can do it from another script, save it, and load it from here, using --tokenizer_name." + ) + + # Preprocessing the datasets. + # First we tokenize all the texts. + # since this will be pickled to avoid _LazyModule error in Hasher force logger loading before tokenize_function + tok_logger = transformers.utils.logging.get_logger( + "transformers.tokenization_utils_base" + ) + + def tokenize_function(examples): + with CaptureLogger(tok_logger) as cl: + output = tokenizer(examples["text"]) + # clm input could be much much longer than block_size + if "Token indices sequence length is longer than the" in cl.out: + tok_logger.warning( + "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits" + " before being passed to the model." + ) + return output + + if data_args.block_size is None: + block_size = tokenizer.model_max_length + if block_size > 1024: + logger.warning( + "The chosen tokenizer supports a `model_max_length` that is longer than the default `block_size` value" + " of 1024. If you would like to use a longer `block_size` up to `tokenizer.model_max_length` you can" + " override this default with `--block_size xxx`." + ) + block_size = 1024 + else: + if data_args.block_size > tokenizer.model_max_length: + logger.warning( + f"The block_size passed ({data_args.block_size}) is larger than the maximum length for the model" + f"({tokenizer.model_max_length}). Using block_size={tokenizer.model_max_length}." + ) + block_size = min(data_args.block_size, tokenizer.model_max_length) + + # Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size. + def group_texts(examples): + # Concatenate all texts. + concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()} + total_length = len(concatenated_examples[list(examples.keys())[0]]) + # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can + # customize this part to your needs. + if total_length >= block_size: + total_length = (total_length // block_size) * block_size + # Split by chunks of max_len. + result = { + k: [t[i : i + block_size] for i in range(0, total_length, block_size)] + for k, t in concatenated_examples.items() + } + result["labels"] = result["input_ids"].copy() + return result + + with training_args.main_process_first(desc="dataset map tokenization and grouping"): + lm_datasets = [] + path = Path(data_args.dataset_dir) + files = [file.name for file in path.glob("*.txt")] + if training_args.debug_mode is True: + files = [files[0]] + for idx, file in enumerate(files): + data_file = os.path.join(path, file) + filename = "".join(file.split(".")[:-1]) + cache_path = os.path.join(data_args.data_cache_dir, filename) + os.makedirs(cache_path, exist_ok=True) + try: + processed_dataset = datasets.load_from_disk( + cache_path, keep_in_memory=False + ) + logger.info(f"training datasets-{filename} has been loaded from disk") + except Exception: + cache_dir = os.path.join(data_args.data_cache_dir, filename + "_text") + os.makedirs(cache_dir, exist_ok=True) + raw_dataset = load_dataset( + "text", + data_files=data_file, + cache_dir=cache_dir, + keep_in_memory=False, + ) + logger.info(f"{file} has been loaded") + tokenized_dataset = raw_dataset.map( + tokenize_function, + batched=True, + num_proc=data_args.preprocessing_num_workers, + remove_columns="text", + load_from_cache_file=True, + keep_in_memory=False, + cache_file_names={ + k: os.path.join(cache_dir, "tokenized.arrow") + for k in raw_dataset + }, + desc="Running tokenizer on dataset", + ) + grouped_datasets = tokenized_dataset.map( + group_texts, + batched=True, + num_proc=data_args.preprocessing_num_workers, + load_from_cache_file=True, + keep_in_memory=False, + cache_file_names={ + k: os.path.join(cache_dir, "grouped.arrow") + for k in tokenized_dataset + }, + desc=f"Grouping texts in chunks of {block_size}", + ) + processed_dataset = grouped_datasets + processed_dataset.save_to_disk(cache_path) + if idx == 0: + lm_datasets = processed_dataset["train"] + else: + assert ( + lm_datasets.features.type + == processed_dataset["train"].features.type + ) + lm_datasets = concatenate_datasets( + [lm_datasets, processed_dataset["train"]] + ) + + lm_datasets = lm_datasets.train_test_split( + test_size=data_args.validation_split_percentage + ) + + if training_args.do_train: + train_dataset = lm_datasets["train"] + if data_args.max_train_samples is not None: + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) + logger.info(f"Num train_samples {len(train_dataset)}") + logger.info("training example:") + logger.info(tokenizer.decode(train_dataset[0]["input_ids"])) + if training_args.do_eval: + eval_dataset = lm_datasets["test"] + if data_args.max_eval_samples is not None: + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) + logger.info(f"Num eval_samples {len(eval_dataset)}") + logger.info("training example:") + logger.info(tokenizer.decode(eval_dataset[0]["input_ids"])) + + if model_args.model_name_or_path: + torch_dtype = ( + model_args.torch_dtype + if model_args.torch_dtype in ["auto", None] + else getattr(torch, model_args.torch_dtype) + ) + model = LlamaForCausalLM.from_pretrained( + model_args.model_name_or_path, + from_tf=bool(".ckpt" in model_args.model_name_or_path), + config=config, + cache_dir=model_args.cache_dir, + revision=model_args.model_revision, + use_auth_token=True if model_args.use_auth_token else None, + torch_dtype=torch_dtype, + low_cpu_mem_usage=True, + ) + else: + model = AutoModelForCausalLM.from_config(config) + n_params = sum({p.data_ptr(): p.numel() for p in model.parameters()}.values()) + logger.info( + f"Training new model from scratch - Total size={n_params/2**20:.2f}M params" + ) + + model_vocab_size = model.get_output_embeddings().weight.size(0) + if not ( + (model_vocab_size == 32000 and len(tokenizer) == 49953) + or (model_vocab_size == 32000 and len(tokenizer) == 32000) + or (model_vocab_size == 49953 and len(tokenizer) == 49953) + or (model_vocab_size == 49954 and len(tokenizer) == 49954) + ): + raise ValueError( + f"The combination of base model (size: {model_vocab_size}) and tokenizer (size: {len(tokenizer)}) is not a valid configuration. Please check our project wiki for further information. \n" + "Valid configurations (base model / tokenizer):\n" + "- Continue pre-training original LLaMA: 32000 / 32000 \n" + "- Pre-training Chinese LLaMA based on original LLaMA: 32000 / 49953 \n" + "- Continue pre-training Chinese LLaMA: 49953 / 49953 \n" + "- Continue pre-training Chinese Alpaca: 49954 / 49954 \n" + ) + + model.resize_token_embeddings(len(tokenizer)) + if training_args.peft_path is not None: + logger.info("Peft from pre-trained model") + model = PeftModel.from_pretrained(model, training_args.peft_path) + else: + logger.info("Init new peft model") + target_modules = training_args.trainable.split(",") + modules_to_save = training_args.modules_to_save + if modules_to_save is not None: + modules_to_save = modules_to_save.split(",") + lora_rank = training_args.lora_rank + lora_dropout = training_args.lora_dropout + lora_alpha = training_args.lora_alpha + logger.info(f"target_modules: {target_modules}") + logger.info(f"lora_rank: {lora_rank}") + peft_config = LoraConfig( + task_type=TaskType.CAUSAL_LM, + target_modules=target_modules, + inference_mode=False, + r=lora_rank, + lora_alpha=lora_alpha, + lora_dropout=lora_dropout, + modules_to_save=modules_to_save, + ) + model = get_peft_model(model, peft_config) + model.print_trainable_parameters() + old_state_dict = model.state_dict + model.state_dict = ( + lambda self, *_, **__: get_peft_model_state_dict(self, old_state_dict()) + ).__get__(model, type(model)) + + # Initialize our Trainer + trainer = Trainer( + model=model, + args=training_args, + train_dataset=train_dataset if training_args.do_train else None, + eval_dataset=eval_dataset if training_args.do_eval else None, + tokenizer=tokenizer, + data_collator=fault_tolerance_data_collator, + compute_metrics=compute_metrics + if training_args.do_eval and not is_torch_tpu_available() + else None, + preprocess_logits_for_metrics=logits_argmax + if training_args.do_eval and not is_torch_tpu_available() + else None, + ) + trainer.add_callback(SavePeftModelCallback("lora_model")) + # Training + if training_args.do_train: + checkpoint = None + if training_args.resume_from_checkpoint is not None: + checkpoint = training_args.resume_from_checkpoint + elif last_checkpoint is not None: + checkpoint = last_checkpoint + train_result = trainer.train(resume_from_checkpoint=checkpoint) + + metrics = train_result.metrics + + max_train_samples = ( + data_args.max_train_samples + if data_args.max_train_samples is not None + else len(train_dataset) + ) + metrics["train_samples"] = min(max_train_samples, len(train_dataset)) + + trainer.log_metrics("train", metrics) + trainer.save_metrics("train", metrics) + trainer.save_state() + + # Evaluation + if training_args.do_eval: + logger.info("*** Evaluate ***") + + metrics = trainer.evaluate() + + max_eval_samples = ( + data_args.max_eval_samples + if data_args.max_eval_samples is not None + else len(eval_dataset) + ) + metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset)) + try: + perplexity = math.exp(metrics["eval_loss"]) + except OverflowError: + perplexity = float("inf") + metrics["perplexity"] = perplexity + + trainer.log_metrics("eval", metrics) + trainer.save_metrics("eval", metrics) + + +if __name__ == "__main__": + main() diff --git a/smoe/metrics/__init__.py b/smoe/metrics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/smoe/metrics/accuracy.py b/smoe/metrics/accuracy.py new file mode 100644 index 0000000..604126c --- /dev/null +++ b/smoe/metrics/accuracy.py @@ -0,0 +1,23 @@ +from sklearn.metrics import accuracy_score + + +def accuracy(predictions, references, normalize=True, sample_weight=None): + return { + "accuracy": float( + accuracy_score( + references, + predictions, + normalize=normalize, + sample_weight=sample_weight, + ) + ) + } + + +def compute_metrics(eval_preds): + preds, labels = eval_preds + # preds have the same shape as the labels, after the argmax(-1) has been calculated + # by preprocess_logits_for_metrics but we need to shift the labels + labels = labels[:, 1:].reshape(-1) + preds = preds[:, :-1].reshape(-1) + return accuracy(predictions=preds, references=labels) diff --git a/smoe/metrics/preprocess.py b/smoe/metrics/preprocess.py new file mode 100644 index 0000000..c1684ca --- /dev/null +++ b/smoe/metrics/preprocess.py @@ -0,0 +1,9 @@ +import torch + + +def logits_argmax(logits: torch.Tensor | tuple[torch.Tensor], labels): + if isinstance(logits, tuple): + # Depending on the model and config, logits may contain extra tensors, + # like past_key_values, but logits always come first + logits = logits[0] + return logits.argmax(dim=-1) diff --git a/smoe/utils/__init__.py b/smoe/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/smoe/utils/config.py b/smoe/utils/config.py new file mode 100644 index 0000000..3375307 --- /dev/null +++ b/smoe/utils/config.py @@ -0,0 +1,233 @@ +import os +import sys +from dataclasses import dataclass, field +from typing import Optional, Type, TypeVar + +from transformers import MODEL_FOR_CAUSAL_LM_MAPPING, TrainingArguments +from transformers.utils.versions import require_version +from transformers import HfArgumentParser + + +MODEL_CONFIG_CLASSES = list(MODEL_FOR_CAUSAL_LM_MAPPING.keys()) +MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES) + + +@dataclass +class ModelArguments: + """ + Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch. + """ + + model_name_or_path: Optional[str] = field( + default=None, + metadata={ + "help": ( + "The model checkpoint for weights initialization.Don't set if you want to train a model from scratch." + ) + }, + ) + tokenizer_name_or_path: Optional[str] = field( + default=None, + metadata={ + "help": ( + "The tokenizer for weights initialization.Don't set if you want to train a model from scratch." + ) + }, + ) + model_type: Optional[str] = field( + default=None, + metadata={ + "help": "If training from scratch, pass a model type from the list: " + + ", ".join(MODEL_TYPES) + }, + ) + config_overrides: Optional[str] = field( + default=None, + metadata={ + "help": ( + "Override some existing default config settings when a model is trained from scratch. Example: " + "n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index" + ) + }, + ) + config_name: Optional[str] = field( + default=None, + metadata={ + "help": "Pretrained config name or path if not the same as model_name" + }, + ) + tokenizer_name: Optional[str] = field( + default=None, + metadata={ + "help": "Pretrained tokenizer name or path if not the same as model_name" + }, + ) + cache_dir: Optional[str] = field( + default=None, + metadata={ + "help": "Where do you want to store the pretrained models downloaded from huggingface.co" + }, + ) + use_fast_tokenizer: bool = field( + default=True, + metadata={ + "help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not." + }, + ) + model_revision: str = field( + default="main", + metadata={ + "help": "The specific model version to use (can be a branch name, tag name or commit id)." + }, + ) + use_auth_token: bool = field( + default=False, + metadata={ + "help": ( + "Will use the token generated when running `huggingface-cli login` (necessary to use this script " + "with private models)." + ) + }, + ) + torch_dtype: Optional[str] = field( + default=None, + metadata={ + "help": ( + "Override the default `torch.dtype` and load the model under this dtype. If `auto` is passed, the " + "dtype will be automatically derived from the model's weights." + ), + "choices": ["auto", "bfloat16", "float16", "float32"], + }, + ) + + def __post_init__(self): + if self.config_overrides is not None and ( + self.config_name is not None or self.model_name_or_path is not None + ): + raise ValueError( + "--config_overrides can't be used in combination with --config_name or --model_name_or_path" + ) + + +@dataclass +class DataArguments: + """ + Arguments pertaining to what data we are going to input our model for training and eval. + """ + + dataset_dir: Optional[str] = field( + default=None, + metadata={"help": "The name of the dataset to use (via the datasets library)."}, + ) + dataset_config_name: Optional[str] = field( + default=None, + metadata={ + "help": "The configuration name of the dataset to use (via the datasets library)." + }, + ) + train_file: Optional[str] = field( + default=None, metadata={"help": "The input training data file (a text file)."} + ) + validation_file: Optional[str] = field( + default=None, + metadata={ + "help": "An optional input evaluation data file to evaluate the perplexity on (a text file)." + }, + ) + max_train_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ) + }, + ) + max_eval_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of evaluation examples to this " + "value if set." + ) + }, + ) + streaming: bool = field(default=False, metadata={"help": "Enable streaming mode"}) + block_size: Optional[int] = field( + default=None, + metadata={ + "help": ( + "Optional input sequence length after tokenization. " + "The training dataset will be truncated in block of this size for training. " + "Default to the model max input length for single sentence inputs (take into account special tokens)." + ) + }, + ) + overwrite_cache: bool = field( + default=False, + metadata={"help": "Overwrite the cached training and evaluation sets"}, + ) + validation_split_percentage: Optional[float] = field( + default=0.05, + metadata={ + "help": "The percentage of the train set used as validation set in case there's no validation split" + }, + ) + preprocessing_num_workers: Optional[int] = field( + default=None, + metadata={"help": "The number of processes to use for the preprocessing."}, + ) + keep_linebreaks: bool = field( + default=True, + metadata={"help": "Whether to keep line breaks when using TXT files or not."}, + ) + data_cache_dir: Optional[str] = field( + default="./", metadata={"help": "The datasets processed stored"} + ) + + def __post_init__(self): + if self.streaming: + require_version( + "datasets>=2.0.0", "The streaming feature requires `datasets>=2.0.0`" + ) + + +@dataclass +class LoraTrainingArguments(TrainingArguments): + trainable: Optional[str] = field(default="q_proj,v_proj") + lora_rank: Optional[int] = field(default=8) + lora_dropout: Optional[float] = field(default=0.1) + lora_alpha: Optional[float] = field(default=32.0) + modules_to_save: Optional[str] = field(default=None) + debug_mode: Optional[bool] = field(default=False) + peft_path: Optional[str] = field(default=None) + + +Arguments = TypeVar("Arguments") + + +def parse_args(*args: Type[Arguments]) -> tuple[Arguments, ...]: + """ + Parse arguments from different argument dataclasses + + Example: + >>> model_args, data_args, train_args = parse_args(ModelArguments, DataArguments, TrainingArguments) + """ + parser = HfArgumentParser(args) + if len(sys.argv) == 2: + if sys.argv[1].endswith(".json"): + # If we pass only one argument to the script and it's the path to a json file, + # let's parse it to get our arguments. + arg_tuple = parser.parse_json_file( + json_file=os.path.abspath(sys.argv[1]) + ) + elif sys.argv[1].endswith(".yaml") or sys.argv[1].endswith(".yml"): + arg_tuple = parser.parse_yaml_file( + yaml_file=os.path.abspath(sys.argv[1]) + ) + else: + raise ValueError(f"Only yaml, yml, and json config files are supported, got {sys.argv[1]}") + else: + arg_tuple = parser.parse_args_into_dataclasses() + + return arg_tuple diff --git a/smoe/utils/extract_text_from_jsonl.py b/smoe/utils/extract_text_from_jsonl.py new file mode 100644 index 0000000..40410f0 --- /dev/null +++ b/smoe/utils/extract_text_from_jsonl.py @@ -0,0 +1,35 @@ +""" +Extract texts from jsonlines file. + +Example: + $ python -m smoe.utils.extract_text_from_jsonl -c content -i resources/redpajama/commoncrawl.jsonl -o resources/redpajama-processed/commoncrawl.txt +""" + +import json +import argparse + + +def get_parser(): + parser = argparse.ArgumentParser() + parser.add_argument( + "-c", "--column_name", default="content", help="text column name" + ) + parser.add_argument("-i", "--input_filepath", help="filepath with text to tokenize") + parser.add_argument("-o", "--output_filepath", help="output filepath") + args = parser.parse_args() + return args + + +def extract_text(): + args = get_parser() + + with open(args.input_filepath, "r", encoding="utf8") as fin: + with open(args.output_filepath, "w", encoding="utf8") as fout: + for line in fin: + ins = json.loads(line) + text = ins[args.column_name] + fout.write(f"{text.strip()}\n") + + +if __name__ == "__main__": + extract_text() diff --git a/smoe/utils/logging.py b/smoe/utils/logging.py new file mode 100644 index 0000000..0d7d75f --- /dev/null +++ b/smoe/utils/logging.py @@ -0,0 +1,43 @@ +import sys +import logging + +import datasets +import transformers +from transformers import TrainingArguments + + +# Setup logging +logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + handlers=[logging.StreamHandler(sys.stdout)], +) + +transformers.utils.logging.enable_default_handler() +transformers.utils.logging.enable_explicit_format() +transformers.tokenization_utils.logging.set_verbosity_warning() + + +def set_logging(should_log, log_level): + if should_log: + # The default of training_args.log_level is passive, so we set log level at info here to have that default. + transformers.utils.logging.set_verbosity_info() + + datasets.utils.logging.set_verbosity(log_level) + transformers.utils.logging.set_verbosity(log_level) + + +def get_logger(name, log_level = None): + logger = logging.getLogger(name) + if log_level: + logger.setLevel(log_level) + return logger + + +def get_logger_from_training_args(name: str, training_args: TrainingArguments): + should_log = training_args.should_log + log_level = training_args.get_process_log_level() + set_logging(should_log, log_level) + logger = get_logger(name, log_level=log_level) + return logger diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..a552a07 --- /dev/null +++ b/tox.ini @@ -0,0 +1,21 @@ +[flake8] +ignore= + # line length + E501, + # whitespace before ':' + E203, + # line break before binary operator + W503, + # import but not used + F401 +exclude = + # No need to traverse our git directory + .git, + # There's no value in checking cache directories + __pycache__, + # This contains our built documentation + build, + # This contains builds of flake8 that we don't want to check + dist, + bak, + data