[QEff.finetuning] Adding sample config and ReadMe file

QEfficient/finetune/experimental/configs/default_config.yaml

@@ -0,0 +1,64 @@
+# -----------------------------------------------------------------------------
+#
+# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries.
+# SPDX-License-Identifier: BSD-3-Clause
+#
+# -----------------------------------------------------------------------------
+
+# Model configuration
+model:
+  model_type: "hf"  # Hugging Face model
+  auto_class_name: "AutoModelForCausalLM"
+  model_name: "HuggingFaceTB/SmolLM-135M"  # Pretrained model name
+  use_peft: true
+  peft_config:
+    lora_r: 8
+    lora_alpha: 16
+    target_modules: ["q_proj", "v_proj"]
+    task_type: "CAUSAL_LM"  # Options: CAUSAL_LM, SEQ_2_SEQ_LM, etc.
+    peft_type: "LORA"  # Options: LORA, IA3, etc.
+
+# Dataset configuration
+dataset:
+  dataset_type: "seq_completion"
+  dataset_name: "knkarthick/samsum"
+  prompt_template: "Summarize the following conversation:\n\n{'dialogue'}\n\nSummary:\n"
+  completion_template: "{summary}"
+  train_split: "train"
+
+# Training configuration
+training:
+  type: "sft"
+  output_dir: "./training_results"
+  eval_strategy: "epoch"
+  gradient_accumulation_steps: 1
+  num_train_epochs: 1
+  max_steps: -1
+  save_strategy: "epoch"
+  torch_dtype: "fp16"
+
+  # Uncomment if running in Notebook
+  # disable_tqdm: True
+
+  # Uncomment and populate to resume training
+  # resume_from_checkpoint: "./abc"
+  # restore_callback_states_from_checkpoint: True
+
+  gradient_checkpointing: False
+  torch_compile: True
+
+# Optimizer configuration
+optimizers:
+  optimizer_name: "adamw"
+  lr: 5e-5
+  weight_decay: 0.01
+
+scheduler:
+  scheduler_name: "cosine"
+  warmup_steps: 100   # warmup_steps or warmup_ratio
+
+callbacks:
+  early_stopping:
+    early_stopping_patience: 3
+    early_stopping_threshold: 0.001
+  tensorboard:

QEfficient/finetune/experimental/core/component_registry.py

@@ -208,3 +208,31 @@ def create_model(model_type: str, model_name: str, **kwargs) -> any:
             raise ValueError(f"Unknown model: {model_type}. Available: {registry.list_models()}")
         model_instance = model_class.create(model_name, **kwargs)
         return model_instance
+
+    def create_trainer_config(name: str, **dependencies) -> tuple:
+        """
+        Create trainer configuration based on registered trainer modules.
+
+        Args:
+            name: Name of the trainer type
+            **dependencies: Any dependencies needed to configure the trainer
+
+        Returns:
+            tuple: (trainer_class, args_class, additional_kwargs)
+        """
+        config = registry.get_trainer_module(name)
+
+        # Process required kwargs based on available dependencies
+        additional_kwargs = {}
+        for kwarg, default in config["required_kwargs"].items():
+            if kwarg in dependencies:
+                additional_kwargs[kwarg] = dependencies[kwarg]
+            elif default != "REQUIRED":
+                additional_kwargs[kwarg] = default
+
+        # Check for missing required arguments
+        for kwarg, default in config["required_kwargs"].items():
+            if kwarg not in additional_kwargs and default == "REQUIRED":
+                raise ValueError(f"Required argument '{kwarg}' not provided for trainer '{name}'")
+
+        return config["trainer_cls"], config["args_cls"], additional_kwargs

QEfficient/finetune/experimental/core/config_manager.py

@@ -18,8 +18,6 @@
 import yaml
 from transformers.hf_argparser import HfArgumentParser
 
-from QEfficient.finetune.experimental.core.component_registry import registry
-
 
 @dataclass
 class OptimizerConfig:
@@ -73,6 +71,10 @@ class DatasetConfig:
         default="knkarthick/samsum",
         metadata={"help": "The name or path of the dataset."},
     )
+    json_file_path: str = field(
+        default=None,
+        metadata={"help": "Path to a custom JSON file containing the dataset."},
+    )
     dataset_subset: str = field(
         default="default",
         metadata={"help": "The subset of the dataset to use, if applicable."},
@@ -412,7 +414,7 @@ class TrainingConfig:
         metadata={"help": "DDP configuration dictionary."},
     )
     use_cpu: Optional[bool] = field(
-        default=None,
+        default=False,
         metadata={"help": "Whether to explicitly run training on CPU."},
     )
     resume_from_checkpoint: Optional[str] = field(
@@ -722,32 +724,3 @@ def __getattr__(self, name: str) -> Any:
         if hasattr(self.config, name):
             return getattr(self.config, name)
         raise AttributeError(f"'{self.__class__.__name__}' object has no attribute '{name}'")
-
-
-def create_trainer_config(name: str, **dependencies) -> tuple:
-    """
-    Create trainer configuration based on registered trainer modules.
-
-    Args:
-        name: Name of the trainer type
-        **dependencies: Any dependencies needed to configure the trainer
-
-    Returns:
-        tuple: (trainer_class, args_class, additional_kwargs)
-    """
-    config = registry.get_trainer_module(name)
-
-    # Process required kwargs based on available dependencies
-    additional_kwargs = {}
-    for kwarg, default in config["required_kwargs"].items():
-        if kwarg in dependencies:
-            additional_kwargs[kwarg] = dependencies[kwarg]
-        elif default != "REQUIRED":
-            additional_kwargs[kwarg] = default
-
-    # Check for missing required arguments
-    for kwarg, default in config["required_kwargs"].items():
-        if kwarg not in additional_kwargs and default == "REQUIRED":
-            raise ValueError(f"Required argument '{kwarg}' not provided for trainer '{name}'")
-
-    return config["trainer_cls"], config["args_cls"], additional_kwargs

QEfficient/finetune/experimental/core/optimizer.py

@@ -13,9 +13,9 @@
 
 from QEfficient.finetune.experimental.core.component_registry import registry
 
-registry.optimizer("Adam")(optim.Adam)
-registry.optimizer("AdamW")(optim.AdamW)
-registry.optimizer("SGD")(optim.SGD)
+registry.optimizer("adam")(optim.Adam)
+registry.optimizer("adamw")(optim.AdamW)
+registry.optimizer("sgd")(optim.SGD)
 
 
 def prepare_optimizer(opt_config):

QEfficient/finetune/experimental/tests/test_config.yaml

@@ -100,5 +100,4 @@ callbacks:
   early_stopping:
     early_stopping_patience: 3
     early_stopping_threshold: 0.001
-  tensorboard:
-
+  tensorboard:

QEfficient/finetune/experimental/tests/test_optimizer.py

@@ -15,42 +15,42 @@
 from QEfficient.finetune.experimental.core.optimizer import prepare_optimizer
 
 OPTIMIZER_CONFIGS = {
-    "Adam": {
-        "optimizer_name": "Adam",
+    "adam": {
+        "optimizer_name": "adam",
         "opt_cls": optim.Adam,
         "lr": 1e-4,
         "weight_decay": 0.01,
         "betas": (0.9, 0.999),
         "eps": 1e-8,
         "amsgrad": False,
     },
-    "AdamW": {
-        "optimizer_name": "AdamW",
+    "adamw": {
+        "optimizer_name": "adamw",
         "opt_cls": optim.AdamW,
         "lr": 1e-4,
         "weight_decay": 0.01,
         "betas": (0.9, 0.999),
         "eps": 1e-8,
         "amsgrad": False,
     },
-    "SGD": {
-        "optimizer_name": "SGD",
+    "sgd": {
+        "optimizer_name": "sgd",
         "opt_cls": optim.SGD,
         "lr": 1e-4,
         "momentum": 0.9,
         "weight_decay": 0.01,
         "dampening": 0.0,
         "nesterov": False,
     },
-    "RMSprop": {
-        "optimizer_name": "RMSprop",
+    "rmsprop": {
+        "optimizer_name": "rmsprop",
         "opt_cls": optim.RMSprop,
     },
 }
 
 REGISTRY_CONFIG = {
-    "RMSprop": {
-        "optimizer_name": "RMSprop",
+    "rmsprop": {
+        "optimizer_name": "rmsprop",
         "opt_cls": optim.RMSprop,
     },
 }