Skip to content

Commit 8b69289

Browse files
cosminbascacosminbasca
committed
gil.py: add ability to automatically remove the gil for a given truffle library or in general
- auto extract messages from supplied lib
1 parent 9b71040 commit 8b69289

File tree

1 file changed

+126
-28
lines changed

1 file changed

+126
-28
lines changed

‎scripts/gil.py

Lines changed: 126 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,30 @@
5454
r"package\s.*?;",
5555
re.DOTALL | re.MULTILINE | re.UNICODE)
5656

57+
PTRN_GILNODE_ARG = re.compile(
58+
r"(?P<start>,)(?P<arg>.*?@Cached GilNode gil)",
59+
re.MULTILINE | re.UNICODE)
60+
61+
PTRN_REM_GIL_TRY_CATCH = re.compile(
62+
r"(boolean mustRelease = gil\.acquire\(\);\s+)?try\s\{\s(?P<body>.+?)\s+\} finally \{\s+gil\.release\(mustRelease\);\s+\}",
63+
re.DOTALL | re.MULTILINE | re.UNICODE)
64+
65+
PTRN_REM_GIL_ARGS = re.compile(
66+
r'(,\s+)?@((Cached\.)?Exclusive|Shared\("gil"\))\s@Cached GilNode gil',
67+
re.DOTALL | re.MULTILINE | re.UNICODE)
68+
69+
PTRN_REM_GIL_BIND = re.compile(
70+
r'(,\s+)?@Bind.*?\sboolean mustRelease',
71+
re.DOTALL | re.MULTILINE | re.UNICODE)
72+
73+
PTRN_LIB_MSG = re.compile(
74+
r'^ \w(?P<header>.*?)\s(?P<method>[a-zA-Z][a-zA-Z0-9]*)\((?P<args>.*?)\)(?P<throws>\sthrows .*?)?\s\{',
75+
re.MULTILINE | re.UNICODE)
76+
77+
PTRN_LIB_MSG_ABS = re.compile(
78+
r'^ \w(?P<header>.*?)\s(?P<method>[a-zA-Z][a-zA-Z0-9]*)\((?P<args>.*?)\)(?P<throws>\sthrows .*?)?;',
79+
re.MULTILINE | re.UNICODE)
80+
5781
RUNTIME_PACKAGE = "package com.oracle.graal.python.runtime;"
5882
GIL_NODE_IMPORT = "import com.oracle.graal.python.runtime.GilNode;"
5983
CACHED_IMPORT = "import com.oracle.truffle.api.dsl.Cached;"
@@ -126,6 +150,19 @@ def is_fallback(self):
126150
def is_with_gil(self):
127151
return "GilNode gil" in self.source
128152

153+
@property
154+
def is_class(self):
155+
return ' class ' in self.match.group('header')
156+
157+
@property
158+
def name(self):
159+
rv = self.match.group('method')
160+
if self.is_class:
161+
hdr = self.match.group('header').split()
162+
name = hdr[hdr.index('class') + 1]
163+
rv = name[:1].lower() + name[1:]
164+
return rv.strip()
165+
129166
@property
130167
def source_with_gil(self):
131168
# handle varargs ...
@@ -134,7 +171,7 @@ def source_with_gil(self):
134171
_uncached_gil = "GilNode gil = GilNode.getUncached();"
135172
else:
136173
_uncached_gil = ""
137-
_args += ", " if self.args else ""
174+
_args += ",\n " if self.args else ""
138175
if self._shared and ('limit = ' not in self.header or 'limit = "1"' in self.header):
139176
_args += '@Shared("gil")'
140177
else:
@@ -152,13 +189,21 @@ def source_with_gil(self):
152189
}
153190
}""" % (self.header, _args, self.throws, _uncached_gil, self.body.strip())
154191

155-
def apply_gil(self):
156-
return
192+
@property
193+
def source_without_gil(self):
194+
source = self.source
195+
source = re.sub(PTRN_REM_GIL_TRY_CATCH, lambda match: match.group('body'), source, 1)
196+
source = re.sub(PTRN_REM_GIL_ARGS, "", source, 1)
197+
source = re.sub(PTRN_REM_GIL_BIND, "", source, 1)
198+
return source
157199

158200
def __str__(self):
159201
return "START: {}, ARGS {}:{}, BODY_START: {}, STOP: {}, CONTENT:\n {}".format(
160202
self._start, self._args_start, self._args_end, self._body_start, self._end, self.source)
161203

204+
def __repr__(self):
205+
return 'Message({})'.format(self.name)
206+
162207

163208
def message_is_class(match):
164209
return ' class ' in match.group('header')
@@ -206,62 +251,113 @@ def file_names_filter(f_name, names):
206251
return False
207252

208253

209-
def main(sources, add=True, dry_run=True, check_style=True, single_source=False, source_filter=None,
210-
ignore_filter=None, count=False, sharing=False):
254+
def fix_gilnode_arg(source):
255+
def repl(match):
256+
return match.group("start") + "\n" + match.group("arg")
257+
return re.sub(PTRN_GILNODE_ARG, repl, source)
258+
259+
260+
def get_lib_messages(lib, files):
261+
if lib is None:
262+
return None
263+
lib_file = next(f for f in files if lib in f)
264+
print("got lib source: {}".format(lib_file))
265+
with open(lib_file, 'r') as SRC:
266+
src = SRC.read()
267+
messages = set()
268+
for m in re.finditer(PTRN_LIB_MSG, src):
269+
messages.add(m.group('method'))
270+
for m in re.finditer(PTRN_LIB_MSG_ABS, src):
271+
messages.add(m.group('method'))
272+
return messages
273+
274+
275+
def main(sources, add=True, lib=None, dry_run=True, check_style=True, single_source=False, source_filter=None,
276+
ignore_filter=None, count=False, sharing=False, fix_style=False):
211277
files = glob.glob("{}**/*.java".format(sources), recursive=True)
278+
lib_messages = get_lib_messages(lib, files)
279+
if lib:
280+
from pprint import pprint
281+
print("[{}] messages: ".format(lib))
282+
pprint(lib_messages)
283+
212284
if ignore_filter:
213285
files = list(filter(lambda f: not file_names_filter(f, ignore_filter), files))
214286
if source_filter and not count:
215287
files = list(filter(lambda f: file_names_filter(f, source_filter), files))
216288

289+
remove = not add
217290
cnt = 0
218291
for java_file in files:
219292
with open(java_file, 'r+') as SRC:
220293
source = SRC.read()
221-
if add:
294+
if fix_style:
295+
if "GilNode" in source:
296+
print("[process] {}".format(java_file))
297+
source = fix_gilnode_arg(source)
298+
SRC.seek(0)
299+
SRC.write(source)
300+
continue
301+
else:
222302
messages, shared = get_messages(source, PTRN_MESSAGE, sharing=sharing)
223303
if len(messages) > 0:
224-
if 'GilNode gil' in source or SKIP_GIL in source:
225-
print("[skipping] {}".format(java_file))
226-
continue
227-
228304
if count:
229305
cnt += 1
230306
continue
231307

232308
print("[process] dry run: {}, add: {}. messages: {}, {}".format(
233309
dry_run, add, len(messages), java_file))
234-
source_with_gil = []
310+
311+
def get_mod_source(msg):
312+
return msg.source_with_gil if add else msg.source_without_gil
313+
314+
if (add and 'GilNode gil' in source) or \
315+
(remove and 'GilNode gil' not in source) or \
316+
SKIP_GIL in source:
317+
print("[skipping] {}".format(java_file))
318+
continue
319+
320+
if remove and '@ExportLibrary({}.class)'.format(lib) not in source:
321+
print("[skipping] {}".format(java_file))
322+
continue
323+
324+
if lib:
325+
messages = list(filter(lambda m: m.name in lib_messages and m.is_with_gil, messages))
326+
print("process messages: ", messages)
327+
328+
if len(messages) == 0:
329+
continue
330+
331+
_src_parts = []
235332
m = messages[0]
236333
if len(messages) == 1:
237-
source_with_gil = [source[:m.start], m.source_with_gil, source[m.end:]]
334+
_src_parts = [source[:m.start], get_mod_source(m), source[m.end:]]
238335
else:
239-
source_with_gil.append(source[:m.start])
336+
_src_parts.append(source[:m.start])
240337
for m1, m2 in zip(messages[:-1], messages[1:]):
241-
source_with_gil.append(m1.source_with_gil)
242-
source_with_gil.append(source[m1.end: m2.start])
243-
source_with_gil.append(m2.source_with_gil)
244-
source_with_gil.append(source[m2.end:])
338+
_src_parts.append(get_mod_source(m1))
339+
_src_parts.append(source[m1.end: m2.start])
340+
_src_parts.append(get_mod_source(m2))
341+
_src_parts.append(source[m2.end:])
342+
343+
modified_source = ''.join(_src_parts)
344+
if add:
345+
modified_source = add_import(modified_source, shared=shared)
245346

246-
source_with_gil = ''.join(source_with_gil)
247-
source_with_gil = add_import(source_with_gil, shared=shared)
248347
if dry_run:
249-
print(source_with_gil)
348+
print(modified_source)
250349
return
251350
else:
351+
SRC.truncate(0)
252352
SRC.seek(0)
253-
SRC.write(source_with_gil)
353+
if modified_source:
354+
SRC.write(modified_source)
254355
if single_source:
255356
break
256-
else:
257-
print("removal of the GIL not yet supported")
258-
return
259357

260358
if count:
261359
print("TO PROCESS: {} files".format(cnt))
262360
if check_style and not count:
263-
# running the checkstyle gate (twice)
264-
# for i in range(2):
265361
os.system("mx python-gate --tags style,python-license")
266362

267363

@@ -271,14 +367,16 @@ def main(sources, add=True, dry_run=True, check_style=True, single_source=False,
271367
action="store_true")
272368
parser.add_argument("--count", help="count how many files may need the GIL", action="store_true")
273369
parser.add_argument("--remove", help="remove the GIL", action="store_true")
370+
parser.add_argument("--lib", type=str, help="the internal library for which messages to remove the GIL")
274371
parser.add_argument("--no_style", help="do not run the style checker", action="store_true")
275372
parser.add_argument("--sharing", help="use @Shared", action="store_true")
276373
parser.add_argument("--single", help="stop after modifying the first source", action="store_true")
277374
parser.add_argument("--filter", type=str, help="filter for source name(s) (comma separated)")
278375
parser.add_argument("--ignore", type=str, help="ignore filter for source name(s) (comma separated)")
376+
parser.add_argument("--fix_style", help="fix GilNode related style issue", action="store_true")
279377
parser.add_argument("sources", type=str, help="location of sources")
280378
args = parser.parse_args()
281379

282-
main(args.sources, add=not args.remove, dry_run=args.dry_run, check_style=not args.no_style,
380+
main(args.sources, add=not args.remove, lib=args.lib, dry_run=args.dry_run, check_style=not args.no_style,
283381
single_source=args.single, source_filter=args.filter, ignore_filter=args.ignore, count=args.count,
284-
sharing=args.sharing)
382+
sharing=args.sharing, fix_style=args.fix_style)

0 commit comments

Comments
 (0)