Skip to content

Commit 555e427

Browse files
OlivierTOMATOOlivierTOMATO
committed
Plotter fixes
1 parent e767409 commit 555e427

14 files changed

+205
-893
lines changed

‎netconfeval/common/model_configs.py

+5
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,11 @@ def _build_mistral_lite_prompt(messages):
118118
'seed': 5000,
119119
}
120120
},
121+
'gpt-3.5-0613': {
122+
'model_name': 'gpt-3.5-turbo-0613',
123+
'type': 'openai',
124+
'args': {}
125+
},
121126
'gpt-4': {
122127
'model_name': 'gpt-4',
123128
'type': 'openai',

‎netconfeval/foundation/langchain/chat_models/hf.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ def _parse_chat_history(history: List[BaseMessage]) -> list[str]:
4040
class ChatHF(BaseChatModel):
4141
model_name: str
4242
max_length: int
43-
temperature: float
43+
temperature: float | None
4444
use_quantization: bool
4545
text_pipeline: Any
4646
prompt_func: Any
@@ -51,6 +51,8 @@ class ChatHF(BaseChatModel):
5151
# and then add it to the path For example: export LD_LIBRARY_PATH="/opt/cuda/11.8.0/targets/x86_64-linux/lib/:$LD_LIBRARY_PATH"
5252
# And then run `python -m bitsandbytes`
5353

54+
#TODO: add parameter, plot by model name
55+
5456
def __init__(self, **kwargs: Any) -> None:
5557
super().__init__(**kwargs)
5658
self.model_name = kwargs.get('model_name', None)

‎netconfeval/step_1_formal_spec_conflict_detection.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ def main(args: argparse.Namespace) -> None:
6969
os.path.abspath(
7070
os.path.join(
7171
args.results_path,
72-
f"log-{args.model}-{'_'.join(policy_types)}-conflict-{results_time}.log"
72+
f"log-{args.model}{"-combined" if args.combined else ""}-{'_'.join(policy_types)}-conflict-{results_time}.log"
7373
)
7474
)
7575
)
@@ -100,7 +100,7 @@ def main(args: argparse.Namespace) -> None:
100100
max_n_requirements = max(args.batch_size) * n_policy_types
101101
w = None
102102

103-
filename = f"result-{args.model}-{'_'.join(policy_types)}-conflict-{results_time}.csv"
103+
filename = f"result-{args.model}-{'_'.join(policy_types)}-conflict{"-combined" if args.combined else ""}-{results_time}.csv"
104104

105105
with open(os.path.join(args.results_path, filename), 'w') as f:
106106
for it in range(0, args.n_runs):

‎netconfeval/step_1_function_call.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ def main(args: argparse.Namespace) -> None:
139139
os.path.abspath(
140140
os.path.join(
141141
args.results_path,
142-
f"log-{args.model}-{'_'.join(policy_types)}-function_{'adhoc' if args.adhoc else 'native'}-{results_time}.log"
142+
f"log-{args.model}-{'adhoc' if args.adhoc else 'native'}-{'_'.join(policy_types)}-function-{results_time}.log"
143143
)
144144
)
145145
)

‎plots/step_1_graph_conflict.py ‎plot/step_1_plot_formal_spec_conflict_detection.py

+38-10
Original file line numberDiff line numberDiff line change
@@ -13,25 +13,53 @@
1313
import matplotlib.pyplot as plt
1414
import numpy as np
1515
from sortedcontainers import SortedSet
16-
from src.utils import strtobool
16+
# TODO: change the location of "combined" in experiments
17+
# TODO: add model_selection
18+
19+
def strtobool(val: str) -> bool:
20+
"""Convert a string representation of truth to true or false.
21+
22+
Args:
23+
val (str): The value to convert. True values are 'y', 'yes', 't', 'true', 'on', and '1';
24+
false values are 'n', 'no', 'f', 'false', 'off', and '0'.
25+
26+
Returns:
27+
bool: The value converted to bool.
28+
29+
Raises:
30+
ValueError: If the specified value is not a valid representation of truth.
31+
"""
32+
val = val.lower()
33+
if val in ('y', 'yes', 't', 'true', 'on', '1'):
34+
return True
35+
elif val in ('n', 'no', 'f', 'false', 'off', '0'):
36+
return False
37+
else:
38+
raise ValueError(f"Invalid truth value `{val}`.")
39+
1740

1841
model2plot = {
19-
"gpt-4-turbo": {
42+
"gpt-4-1106": {
2043
"label": "GPT-4-Turbo",
2144
"color": "#377eb8",
2245
"marker": "o"
2346
},
24-
"gpt-4": {
47+
"gpt-4-": {
2548
"label": "GPT-4",
26-
"color": "#e41a1c",
49+
"color": "#a65628",
2750
"marker": "s"
2851
},
29-
"gpt-3.5-turbo": {
52+
"gpt-3.5-0613": {
3053
"label": "GPT-3.5-Turbo",
31-
"color": "#4daf4a",
54+
"color": "#984ea3",
3255
"marker": ">"
3356
},
34-
"gpt-4-combined": {
57+
# "gpt-4-turbo-function": {
58+
# "label": "gpt-4-turbo-function",
59+
# "color": "#e41a1c",
60+
# "marker": "1"
61+
# },
62+
"gpt-4-1106-combined": {
3563
"label": "GPT-4 (Combined)",
3664
"color": "#984ea3",
3765
"marker": "<"
@@ -141,7 +169,7 @@ def plot_by_requirements(results_path: str, figures_path: str, requirements: Sor
141169
model_list = model2plot.keys()
142170
for model_name in model_list:
143171
results_files_list = glob.glob(
144-
os.path.join(".", results_path, f"result-{model_name}-{requirements_str}-conflict-*.csv"))
172+
os.path.join("../", results_path, f"result-{model_name}-{requirements_str}-conflict-*.csv"))
145173

146174
if len(results_files_list) == 0:
147175
continue
@@ -158,7 +186,7 @@ def plot_by_requirements(results_path: str, figures_path: str, requirements: Sor
158186
print(model2result[model_name]["accuracy"], model2result[model_name]["precision"],
159187
model2result[model_name]["recall"], model2result[model_name]["f1_score"])
160188

161-
base_figures_path = os.path.join(".", figures_path)
189+
base_figures_path = os.path.join(figures_path)
162190
os.makedirs(base_figures_path, exist_ok=True)
163191

164192
for param in ["accuracy", "precision", "recall", "f1_score"]:
@@ -220,4 +248,4 @@ def main(args: argparse.Namespace) -> None:
220248

221249

222250
if __name__ == "__main__":
223-
main(parse_args())
251+
main(parse_args())

‎plots/step_1_heat_map.py ‎plot/step_1_plot_formal_spec_conflict_heatmap.py

+30-10
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
1+
import argparse
12
import csv
3+
import glob
24

35
import matplotlib
46
import numpy as np
57
import seaborn as sns
68
from matplotlib import pyplot as plt
7-
9+
import sys
10+
import os
11+
sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), ".."))
812

913
def heatmap(data_loc: str, fig_path: str) -> None:
1014
results = [[0] * 34 for i in range(34)]
@@ -50,17 +54,33 @@ def heatmap(data_loc: str, fig_path: str) -> None:
5054
fig.savefig(fig_path, format="pdf", bbox_inches='tight')
5155

5256

53-
if __name__ == "__main__":
57+
def main(args: argparse.Namespace) -> None:
5458
matplotlib.rc('font', size=10)
5559
matplotlib.rcParams['hatch.linewidth'] = 0.3
5660
matplotlib.rcParams['pdf.fonttype'] = 42
5761
matplotlib.rcParams['ps.fonttype'] = 42
5862

59-
heatmap(
60-
"evaluation/result_conflict/result_distance/result-gpt-3.5-turbo-loadbalancing_reachability_waypoint-conflict-1700742765.7879624.csv",
61-
"evaluation/result_conflict/result_distance/heatmap-gpt-3.5.pdf"
62-
)
63-
heatmap(
64-
"evaluation/result_conflict/result_distance/result-gpt-4-turbo-loadbalancing_reachability_waypoint-conflict-1700747740.0543778.csv",
65-
"evaluation/result_conflict/result_distance/heatmap-gpt-4.pdf"
66-
)
63+
results_files_list = glob.glob(os.path.join("../", args.results_path, f"*.csv"))
64+
base_figures_path = os.path.join("../plot", args.figures_path)
65+
os.makedirs(base_figures_path, exist_ok=True)
66+
67+
for file in results_files_list:
68+
heatmap(
69+
file,
70+
os.path.join(base_figures_path, "heatmap-gpt-3.5.pdf")
71+
)
72+
# heatmap(
73+
# "evaluation/result_conflict/result_distance/result-gpt-4-turbo-loadbalancing_reachability_waypoint-conflict-1700747740.0543778.csv",
74+
# "evaluation/result_conflict/result_distance/heatmap-gpt-4.pdf"
75+
# )
76+
77+
def parse_args() -> argparse.Namespace:
78+
parser = argparse.ArgumentParser()
79+
parser.add_argument('--results_path', type=str, required=False, default="result")
80+
parser.add_argument('--figures_path', type=str, required=True)
81+
82+
return parser.parse_args()
83+
84+
85+
if __name__ == "__main__":
86+
main(parse_args())

‎plots/step_1_graph_model_compare_ad_hoc.py ‎plot/step_1_plot_formal_spec_translation.py

+50-38
Original file line numberDiff line numberDiff line change
@@ -10,34 +10,6 @@
1010
import numpy as np
1111
from sortedcontainers import SortedSet
1212

13-
model2plot = {
14-
# "gpt-3.5-finetuned": {
15-
# "label": "GPT-3.5-FT",
16-
# "color": "#a65628",
17-
# "marker": "s"
18-
# },
19-
"gpt-3.5-1106-function": {
20-
"label": "GPT-3.5-function",
21-
"color": "#984ea3",
22-
"marker": ">"
23-
},
24-
"gpt-4-turbo-function": {
25-
"label": "gpt-4-turbo-function",
26-
"color": "#e41a1c",
27-
"marker": "1"
28-
},
29-
# "gpt-4": {
30-
# "label": "GPT-4",
31-
# "color": "#ffff33",
32-
# "marker": "+"
33-
# },
34-
"codellama-7b-instruct": {
35-
"label": "CL-7B-Instruct",
36-
"color": "#ff7f00",
37-
"marker": "<"
38-
},
39-
}
40-
4113

4214
def extract_result(file_path: str, model_name: str) -> (dict, dict):
4315
average = {}
@@ -112,15 +84,15 @@ def extract_result(file_path: str, model_name: str) -> (dict, dict):
11284
return to_plot_accuracy, to_plot_cost
11385

11486

115-
def plot_by_requirements(results_path: str, figures_path: str, requirements: SortedSet) -> None:
87+
def plot_by_requirements(results_path: str, figures_path: str, requirements: SortedSet, model2plot) -> None:
88+
# print(model2plot)
89+
# exit()
11690
model2result = {}
11791
requirements_str = "_".join(requirements)
11892

11993
for model_name in model2plot.keys():
120-
# if include not in model_name or 'function' in model_name:
121-
# continue
12294

123-
results_files_list = glob.glob(os.path.join(".", results_path, f"result-{model_name}-ad-hoc-{requirements_str}-*.csv"))
95+
results_files_list = glob.glob(os.path.join("../", results_path, f"result-{model_name}-{requirements_str}-*.csv"))
12496
if results_files_list:
12597
results_file = results_files_list.pop()
12698

@@ -130,7 +102,7 @@ def plot_by_requirements(results_path: str, figures_path: str, requirements: Sor
130102
model2result[model_name]["accuracy"], model2result[model_name]["cost"] = extract_result(results_file,
131103
model_name)
132104

133-
base_figures_path = os.path.join(".", figures_path)
105+
base_figures_path = os.path.join("../plot", figures_path)
134106
os.makedirs(base_figures_path, exist_ok=True)
135107

136108
# Accuracy
@@ -161,6 +133,9 @@ def plot_by_requirements(results_path: str, figures_path: str, requirements: Sor
161133
plt.ylim([-0.1, 1.2])
162134
plt.yticks(np.arange(0, 1.2, 0.25))
163135
plt.xscale('log', base=10)
136+
137+
# print(model2result)
138+
# exit()
164139
x_ticks = list(model2result.values())[0]["accuracy"]["x"]
165140
plt.xticks(x_ticks)
166141
ax.xaxis.set_major_formatter(matplotlib.ticker.ScalarFormatter())
@@ -215,7 +190,7 @@ def parse_args() -> argparse.Namespace:
215190
parser = argparse.ArgumentParser()
216191
parser.add_argument('--results_path', type=str, required=False, default="result")
217192
parser.add_argument('--figures_path', type=str, required=True)
218-
# parser.add_argument('--include', type=str, choices=['gpt', 'codellama'])
193+
parser.add_argument("--models", type=str, choices=["gpt", "codellama"])
219194

220195
return parser.parse_args()
221196

@@ -226,11 +201,48 @@ def main(args: argparse.Namespace) -> None:
226201
matplotlib.rcParams['pdf.fonttype'] = 42
227202
matplotlib.rcParams['ps.fonttype'] = 42
228203

229-
plot_by_requirements(args.results_path, args.figures_path, SortedSet({"reachability"}))
230-
plot_by_requirements(args.results_path, args.figures_path, SortedSet({"reachability", "waypoint"}))
204+
if args.models == "gpt":
205+
model2plot = {
206+
"gpt-4-1106": {
207+
"label": "GPT-4-Turbo",
208+
"color": "#377eb8",
209+
"marker": "o"
210+
},
211+
"gpt-3.5-finetuned": {
212+
"label": "GPT-3.5-FT",
213+
"color": "#a65628",
214+
"marker": "s"
215+
},
216+
"gpt-3.5-0613": {
217+
"label": "GPT-3.5-Turbo",
218+
"color": "#984ea3",
219+
"marker": ">"
220+
},
221+
}
222+
if args.models == "codellama":
223+
model2plot = {
224+
"codellama-13b-instruct": {
225+
"label": "CL-13B-Instruct",
226+
"color": "#ff7f00",
227+
"marker": "<"
228+
},
229+
"codellama-7b-instruct-finetuned": {
230+
"label": "CL-7B-Instruct-FT (QLoRA)",
231+
"color": "#4daf4a",
232+
"marker": ">"
233+
},
234+
"codellama-7b-instruct": {
235+
"label": "CL-7B-Instruct",
236+
"color": "#f781bf",
237+
"marker": "^"
238+
},
239+
}
240+
241+
# plot_by_requirements(args.results_path, args.figures_path, SortedSet({"reachability"}), model2plot)
242+
# plot_by_requirements(args.results_path, args.figures_path, SortedSet({"reachability", "waypoint"}), model2plot)
231243
plot_by_requirements(args.results_path, args.figures_path,
232-
SortedSet({"loadbalancing", "reachability", "waypoint"}))
244+
SortedSet({"loadbalancing", "reachability", "waypoint"}), model2plot)
233245

234246

235247
if __name__ == "__main__":
236-
main(parse_args())
248+
main(parse_args())

‎plots/step_1_graph_function_call_adhoc.py ‎plot/step_1_plot_function_call_ad_hoc.py

+8-8
Original file line numberDiff line numberDiff line change
@@ -10,22 +10,22 @@
1010
from sortedcontainers import SortedSet
1111

1212
model2plot = {
13-
"gpt-3.5-1106-function-ad-hoc": {
13+
"gpt-3.5-1106-adhoc": {
1414
"label": "GPT-3.5-Turbo",
1515
"color": "#984ea3",
1616
"marker": ">"
1717
},
18-
"gpt-4-turbo-function": {
18+
"gpt-4-1106-native": {
1919
"label": "GPT-4-Turbo (Native)",
2020
"color": "#e41a1c",
2121
"marker": "s"
2222
},
23-
"gpt-4-turbo-ad-hoc": {
23+
"gpt-4-turbo-adhoc": {
2424
"label": "GPT-4-Turbo (Ad-hoc)",
2525
"color": "#377eb8",
2626
"marker": "o"
2727
},
28-
"codellama-7b-instruct-ad-hoc": {
28+
"codellama-7b-instruct-adhoc": {
2929
"label": "CL-7B-Instruct",
3030
"color": "#f781bf",
3131
"marker": "^"
@@ -84,7 +84,7 @@ def plot_by_requirements(results_path: str, figures_path: str, requirements: Sor
8484

8585
for model_name in model2plot.keys():
8686
results_files_list = glob.glob(
87-
os.path.join(".", results_path, f"result-{model_name}-{requirements_str}-*.csv")
87+
os.path.join("../", results_path, f"result-{model_name}-{requirements_str}-*.csv")
8888
)
8989
if results_files_list:
9090
results_file = results_files_list.pop()
@@ -150,10 +150,10 @@ def main(args: argparse.Namespace) -> None:
150150
matplotlib.rcParams['pdf.fonttype'] = 42
151151
matplotlib.rcParams['ps.fonttype'] = 42
152152

153-
plot_by_requirements(args.results_path, args.figures_path, SortedSet({"reachability"}))
154-
plot_by_requirements(args.results_path, args.figures_path, SortedSet({"reachability", "waypoint"}))
153+
# plot_by_requirements(args.results_path, args.figures_path, SortedSet({"reachability"}))
154+
# plot_by_requirements(args.results_path, args.figures_path, SortedSet({"reachability", "waypoint"}))
155155
plot_by_requirements(args.results_path, args.figures_path, SortedSet({"loadbalancing", "reachability", "waypoint"}))
156156

157157

158158
if __name__ == "__main__":
159-
main(parse_args())
159+
main(parse_args())

0 commit comments

Comments
 (0)