feat: new format for the controller metrics and operations

Signed-off-by: Harikrishnan Balagopal <[email protected]>

.gitignore

@@ -36,3 +36,6 @@ venv/
 *.bkp
 *.bkp.*
 *bkp*
+
+# Auto generated file
+tuning/_version.py

examples/trainercontroller_configs/loss.yaml

@@ -1,6 +1,6 @@
 controller-metrics:
-  loss:
-    Loss:
+  - name: loss
+    class: Loss
 controllers:
   - name: loss-controller
     triggers:

pyproject.toml

@@ -36,7 +36,8 @@ dependencies = [
 "trl",
 "peft>=0.8.0",
 "datasets>=2.15.0",
-"fire"
+"fire",
+"jsonschema>=4.21.1"
 ]
 
 [project.optional-dependencies]

tests/data/trainercontroller/loss_custom_metric.yaml

@@ -1,6 +1,6 @@
 controller-metrics:
-  testflag:
-    CustomMetric:
+  - name: testflag
+    class: CustomMetric
 controllers:
   - name: loss-controller-custom-metric
     triggers:

tests/data/trainercontroller/loss_custom_operation.yaml

@@ -1,9 +1,9 @@
 controller-metrics:
-  loss:
-    Loss:
+  - name: loss
+    class: Loss
 operations:
-  customoperation:
-    CustomOperation:
+  - name: customoperation
+    class: CustomOperation
 controllers:
   - name: loss-controller-custom-operation
     triggers:

tests/data/trainercontroller/loss_custom_operation_invalid_action.yaml

@@ -1,9 +1,9 @@
 controller-metrics:
-  loss:
-    Loss:
+  - name: loss
+    class: Loss
 operations:
-  customoperation:
-    CustomOperationInvalidAction:
+  - name: customoperation
+    class: CustomOperationInvalidAction
 controllers:
   - name: loss-controller-custom-operation-invalid-action
     triggers:

tests/data/trainercontroller/loss_invalid_metric.yaml

@@ -1,6 +1,6 @@
 controller-metrics:
-  loss:
-    MissingMetricClass:
+  - name: loss
+    class: MissingMetricClass
 controllers:
   - name: loss-controller-invalid-metric
     triggers:

tests/data/trainercontroller/loss_invalid_operation.yaml

@@ -1,6 +1,6 @@
 controller-metrics:
-  loss:
-    Loss:
+  - name: loss
+    class: Loss
 controllers:
   - name: loss-controller-invalid-operation
     triggers:

tests/data/trainercontroller/loss_invalid_operation_action.yaml

@@ -1,6 +1,6 @@
 controller-metrics:
-  loss:
-    Loss:
+  - name: loss
+    class: Loss
 controllers:
   - name: loss-controller-invalid-operation-action
     triggers:

tests/data/trainercontroller/loss_invalid_trigger.yaml

@@ -1,6 +1,6 @@
 controller-metrics:
-  loss:
-    Loss:
+  - name: loss
+    class: Loss
 controllers:
   - name: loss-controller-invalid-trigger
     triggers:

tests/data/trainercontroller/loss_on_threshold.yaml

@@ -1,6 +1,6 @@
 controller-metrics:
-  loss:
-    Loss:
+  - name: loss
+    class: Loss
 controllers:
   - name: loss-controller
     triggers:

tests/data/trainercontroller/loss_with_malicious_input_rule.yaml

@@ -1,6 +1,6 @@
 controller-metrics:
-  loss:
-    Loss:
+  - name: loss
+    class: Loss
 controllers:
   - name: loss-controller-wrong-input-rule
     triggers:

tests/data/trainercontroller/loss_with_malicious_os_rule.yaml

@@ -1,6 +1,6 @@
 controller-metrics:
-  loss:
-    Loss:
+  - name: loss
+    class: Loss
 controllers:
   - name: loss-controller-wrong-os-rule
     triggers: