-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy patharguments.py
177 lines (146 loc) · 12.4 KB
/
arguments.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
import os
from src.comment_argument_parser import CommentArgumentParser
# training_args are passed directly into the TrainingArguments object.
# Below are the most common of the 101 arguments available
# see https://huggingface.co/docs/transformers/v4.35.0/en/main_classes/trainer#transformers.TrainingArguments
training_args = {
"lr_scheduler_type" : "cosine",
"per_device_eval_batch_size" : 2000,
"save_strategy" : "no",
"evaluation_strategy" : "no",
"output_dir" : "out",
}
###
# Argument processing
###
class ArgumentException(Exception):
pass
class _Args(object):
instance = None
def _parse_arguments(self):
to_int_list = lambda s : list( int(x.strip()) for x in s.split(',') if x ) if s is not None else None
to_float_list = lambda s : list( float(x.strip()) for x in s.split(',') if x ) if s is not None else None
to_string_list = lambda s : list( x.strip() for x in s.split(',') if x ) if s is not None else None
parser = CommentArgumentParser("Compare a series of scorefiles", fromfile_prefix_chars='@')
main_group = parser.add_argument_group('Main arguments')
model_group = parser.add_argument_group('Defining the model architecture')
features_group = parser.add_argument_group('Feature extraction')
training_group = parser.add_argument_group('Training constants')
metaparameter_group = parser.add_argument_group('Metaparameters')
main_group.add_argument('-d', '--directory', help="Top level directory", required=True)
main_group.add_argument('-s', '--savefile', default="", help="Filename of (csv) scores file to save (default no save)")
main_group.add_argument('--model', default="model.safetensors", help="Filename to save model (relative to --directory, default model.safetensors)")
main_group.add_argument('--modelpath', help="Full path of model (if set, --model is ignored)")
main_group.add_argument('--scores', default="scores.json", help="Filename of scores file (default scores.json)")
main_group.add_argument('--server', default='on', choices=['on','daemon','off'], help="Start an optuna dashboard server. Use daemon to start in daemon thread (terminates with training), or off for no server")
main_group.add_argument('--database', default="sqlite:///db.sqlite", help="Storage for optuna. Set --database= for no database (implies --server=off)")
model_group.add_argument('--final_layer_bias', action="store_true", help="Train with a bias in the final layer")
model_group.add_argument('--model_seed', type=int, default=0, help="Seed for initialising model (default none)")
model_group.add_argument('--min_first_layer_size', type=int, default=10, help="Minimum number of features in first hidden layer (default 10)")
model_group.add_argument('--max_first_layer_size', type=int, default=1000, help="Maximum number of features in first hidden layer (default 1000)")
model_group.add_argument('--min_second_layer_size', type=int, default=10, help="Minimum number of features in second hidden layer (default 10)")
model_group.add_argument('--max_second_layer_size', type=int, default=1000, help="Maximum number of features in second hidden layer (default 1000)")
features_group.add_argument('--feature_extractor', type=to_string_list, default="ChrisGoringe/vit-large-p14-vision-fp16", help="Model to use for feature extraction")
features_group.add_argument('--hidden_states_used', type=to_int_list, default=None, help="Comma separated list of the hidden states to extract features from (0 is output layer, 1 is last hidden layer etc.)")
features_group.add_argument('--hidden_states_mode', default="default", choices=["default", "join", "average", "weight"], help="Combine multiple layers from feature extractor by join (default), average, or weight")
features_group.add_argument('--fixed_hidden_state_weights', type=to_float_list, default=None, help="When hidden_states_mode=weight, fix the weights and bias (list of float, last entry is bias)")
features_group.add_argument('--fp16_features', action="store_true", help="Store features in fp16")
training_group.add_argument('--loss_model', default='mse', choices=['mse','ab','nll', 'wmse'], help="Loss model (default mse) (mse=mean square error, ab=ab ranking, nll=negative log likelihood, wmse=weighted mse )")
training_group.add_argument('--set_for_scoring', default='eval', choices=['eval', 'full', 'train'], help="Image set to be used for scoring a model when trained (default eval)")
training_group.add_argument('--metric_for_scoring', choices=['mse', 'ab', 'nll', 'wmse', 'spearman', 'pearson', 'accuracy'], help="Metric to be used for scoring a model when trained (default is the loss_model)")
training_group.add_argument('--calculate_ab', action="store_true", help="Calculate ab even if not being used for scoring")
training_group.add_argument('--calculate_mse', action="store_true", help="Calculate mse even if not being used for scoring")
training_group.add_argument('--calculate_spearman', action="store_true", help="Calculate spearman")
training_group.add_argument('--calculate_pearson', action="store_true", help="Calculate pearson")
training_group.add_argument('--calculate_accuracy', action="store_true", help="Calculate accuracy (fraction correct side of accuracy_divider)")
training_group.add_argument('--accuracy_divider', type=float, help="Divider between 'high' and 'low' for accuracy. If not specified the median score is used.")
training_group.add_argument('--normalise_weights', action="store_true", help="If using wmse, normalise the weights to mean of 1.0")
training_group.add_argument('--fraction_for_eval', type=float, default=0.25, help="fraction of images to be reserved for eval (default 0.25)")
training_group.add_argument('--eval_pick_seed', type=int, default=42, help="Seed for random numbers when choosing eval images (default 42)")
training_group.add_argument('--ignore_existing_split', action="store_true", help="Discard existing train/eval split")
metaparameter_group.add_argument('--name', help="Name prefix for Optuna")
metaparameter_group.add_argument('--trials', type=int, default=200, help="Number of metaparameter trials" )
metaparameter_group.add_argument('--sampler', default="CmaEs", choices=['CmaEs', 'random', 'QMC'], help="Metaparameter search algorithm")
metaparameter_group.add_argument('--min_train_epochs', type=int, default=5, help="(default 5)")
metaparameter_group.add_argument('--max_train_epochs', type=int, default=50, help="(default 50)")
metaparameter_group.add_argument('--min_warmup_ratio', type=float, default=0.0, help="(default 0.0)")
metaparameter_group.add_argument('--max_warmup_ratio', type=float, default=0.2, help="(default 0.2)")
metaparameter_group.add_argument('--min_log_lr', type=float, default=-4.5, help="(default -4.5)")
metaparameter_group.add_argument('--max_log_lr', type=float, default=-2.5, help="(default -2.5)")
metaparameter_group.add_argument('--min_log_weight_lr', type=float, default=-4.5, help="(default -4.5)")
metaparameter_group.add_argument('--max_log_weight_lr', type=float, default=-2.5, help="(default -2.5)")
metaparameter_group.add_argument('--min_batch_size', type=int, default=1, help="(default 1)")
metaparameter_group.add_argument('--max_batch_size', type=int, default=100, help="(default 100)")
metaparameter_group.add_argument('--min_dropout', type=float, default=0.0, help="Minimum dropout between two hidden layers (default 0.0)")
metaparameter_group.add_argument('--max_dropout', type=float, default=0.8, help="Maximum dropout between two hidden layers (default 0.8)")
metaparameter_group.add_argument('--min_input_dropout', type=float, default=0.0, help="Minimum dropout between features and first hidden layer (default 0.0)")
metaparameter_group.add_argument('--max_input_dropout', type=float, default=0.8, help="Maximum dropout between features and first hidden layer (default 0.8)")
metaparameter_group.add_argument('--min_output_dropout', type=float, default=0.0, help="Minimum dropout before final projection (default 0.0)")
metaparameter_group.add_argument('--max_output_dropout', type=float, default=0.0, help="Maximum dropout before final projection (default 0.0)")
into={}
namespace, unknowns = parser.parse_known_args()
if unknowns: print(f"\nIgnoring unknown argument(s) {unknowns}")
d = vars(namespace)
into[":Arguments (specified or default)"]=None
for argument in d: into[argument] = d[argument]
into[":Derived arguments"]=None
for argument in list(a[4:] for a in into if a.startswith('min_')):
into[argument] = d[f"min_{argument}"] if d[f"min_{argument}"] == d[f"max_{argument}"] else (d[f"min_{argument}"], d[f"max_{argument}"])
if into['hidden_states_mode']!='weight' or into['fixed_hidden_state_weights'] is not None: into['log_weight_lr'] = 0
into['metric_for_scoring'] = into.get('metric_for_scoring', None) or into['loss_model']
into['parameter_for_scoring'] = f"{into['set_for_scoring']}_{into['metric_for_scoring']}"
into['measures'] = list(o for o in ['ab', 'mse', 'wmse', 'spearman', 'pearson', 'accuracy'] if o==into['loss_model'] or o==into['metric_for_scoring'] or into.get(f"calculate_{o}",False))
into['save_model_path'] = into.get('modelpath',None) or os.path.join(into['directory'], into['model'])
into['score_direction'] = 'maximize' if into['metric_for_scoring'] in ['ab', 'spearman', 'pearson', 'accuracy', ] else 'minimize'
into['output_channels'] = 2 if into['loss_model']=='nll' else 1
into[":Set arguments"]=None
into["training_args"] = training_args
return into
def __init__(self):
self.args = self._parse_arguments()
self.show_args()
self.validate()
self.arg_sets = { "feature_extractor_extras" : ['hidden_states_used','hidden_states_mode','fp16_features',],
"aesthetic_model_extras" : ['hidden_states_used','hidden_states_mode','fixed_hidden_state_weights','model_seed','dropouts','layers','output_channels',],
"trainer_extras" : ['weight_learning_rate'],
}
for k in list(self.args):
if k.startswith(':'): self.args.pop(k)
def __getattr__(self, attr):
return self.get(attr)
def get(self, attr, default=Exception()):
if attr in self.args: return self.args[attr]
if attr in self.arg_sets: return { x : self.args[x] for x in self.arg_sets[attr] }
if isinstance(default, Exception): raise KeyError(attr)
return default
def set(self, key, value):
self.args[key] = value
def show_args(self):
for a in self.args:
if a.startswith(':'):
print(f"\n{a[1:]}\n")
else:
print("{:>30} : {:<40}".format(a, str(self.get(a))))
def validate(self):
if not os.path.isdir(self.directory):
raise ArgumentException( f"{self.directory} doesn't exist or isn't a directory" )
if not os.path.exists(os.path.join(self.directory, self.scores)):
raise ArgumentException(f"{os.path.join(self.directory, self.scores)} not found")
# if self.hidden_states_mode=='weight' and self.fixed_hidden_state_weights is not None and len(self.fixed_hidden_state_weights)!=len(self.hidden_states_used)+1:
# raise ArgumentException(f"Need {len(self.hidden_states_used)+1} values for fixed_hidden_state_weights (weights followed by bias), got {self.fixed_hidden_state_weights}")
if self.get('model',None) and self.get('modelpath',None): print(f"--model ignored because --modelpath was set")
@property
def keys(self):
for a in self.args: yield a
def meta(self, mthd, label:str, rng:tuple|list):
if isinstance(rng,(tuple,list)):
return mthd(label, *rng)
else:
return rng
def meta_list(self, mthd, label:str, rnges:tuple|list):
result = []
for i, rng in enumerate(rnges):
result.append(self.meta(mthd, f"{label}_{i}", rng))
return result
_Args.instance = _Args.instance or _Args()
Args = _Args.instance