Skip to content

Commit f2ab255

Browse files
gh-134873: Fix quadratic complexity in os.path.expandvars()
1 parent 1a89991 commit f2ab255

File tree

5 files changed

+121
-31
lines changed

5 files changed

+121
-31
lines changed

Lib/ntpath.py

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -509,6 +509,71 @@ def expandvars(path):
509509
return res
510510

511511

512+
_varpattern = r"'[^']*'?|%(%|[^%]*%?)|\$(\$|[-\w]+|\{[^}]*\}?)"
513+
_varsub = None
514+
_varsubb = None
515+
516+
def expandvars(path):
517+
"""Expand shell variables of the forms $var, ${var} and %var%.
518+
519+
Unknown variables are left unchanged."""
520+
path = os.fspath(path)
521+
global _varsub, _varsubb
522+
if isinstance(path, bytes):
523+
if b'$' not in path and b'%' not in path:
524+
return path
525+
if not _varsubb:
526+
import re
527+
_varsubb = re.compile(_varpattern.encode(), re.ASCII).sub
528+
sub = _varsubb
529+
percent = b'%'
530+
brace = b'{'
531+
rbrace = b'}'
532+
dollar = b'$'
533+
environ = getattr(os, 'environb', None)
534+
else:
535+
if '$' not in path and '%' not in path:
536+
return path
537+
if not _varsub:
538+
import re
539+
_varsub = re.compile(_varpattern, re.ASCII).sub
540+
sub = _varsub
541+
percent = '%'
542+
brace = '{'
543+
rbrace = '}'
544+
dollar = '$'
545+
environ = os.environ
546+
547+
def repl(m):
548+
lastindex = m.lastindex
549+
if lastindex is None:
550+
return m[0]
551+
name = m[lastindex]
552+
if lastindex == 1:
553+
if name == percent:
554+
return name
555+
if not name.endswith(percent):
556+
return m[0]
557+
name = name[:-1]
558+
else:
559+
if name == dollar:
560+
return name
561+
if name.startswith(brace):
562+
if not name.endswith(rbrace):
563+
return m[0]
564+
name = name[1:-1]
565+
566+
try:
567+
if environ is None:
568+
return os.fsencode(os.environ[os.fsdecode(name)])
569+
else:
570+
return environ[name]
571+
except KeyError:
572+
return m[0]
573+
574+
return sub(repl, path)
575+
576+
512577
# Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B.
513578
# Previously, this function also truncated pathnames to 8+3 format,
514579
# but as this module is called "ntpath", that's obviously wrong!

Lib/posixpath.py

Lines changed: 20 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -284,56 +284,53 @@ def expanduser(path):
284284
# This expands the forms $variable and ${variable} only.
285285
# Non-existent variables are left unchanged.
286286

287-
_varprog = None
288-
_varprogb = None
287+
_varpattern = r'\$(\w+|\{[^}]*\}?)'
288+
_varsub = None
289+
_varsubb = None
289290

290291
def expandvars(path):
291292
"""Expand shell variables of form $var and ${var}. Unknown variables
292293
are left unchanged."""
293294
path = os.fspath(path)
294-
global _varprog, _varprogb
295+
global _varsub, _varsubb
295296
if isinstance(path, bytes):
296297
if b'$' not in path:
297298
return path
298-
if not _varprogb:
299+
if not _varsubb:
299300
import re
300-
_varprogb = re.compile(br'\$(\w+|\{[^}]*\})', re.ASCII)
301-
search = _varprogb.search
301+
_varsubb = re.compile(_varpattern.encode(), re.ASCII).sub
302+
sub = _varsubb
302303
start = b'{'
303304
end = b'}'
304305
environ = getattr(os, 'environb', None)
305306
else:
306307
if '$' not in path:
307308
return path
308-
if not _varprog:
309+
if not _varsub:
309310
import re
310-
_varprog = re.compile(r'\$(\w+|\{[^}]*\})', re.ASCII)
311-
search = _varprog.search
311+
_varsub = re.compile(_varpattern, re.ASCII).sub
312+
sub = _varsub
312313
start = '{'
313314
end = '}'
314315
environ = os.environ
315-
i = 0
316-
while True:
317-
m = search(path, i)
318-
if not m:
319-
break
320-
i, j = m.span(0)
321-
name = m.group(1)
322-
if name.startswith(start) and name.endswith(end):
316+
317+
def repl(m):
318+
name = m[1]
319+
if name.startswith(start):
320+
if not name.endswith(end):
321+
return m[0]
323322
name = name[1:-1]
324323
try:
325324
if environ is None:
326325
value = os.fsencode(os.environ[os.fsdecode(name)])
327326
else:
328327
value = environ[name]
329328
except KeyError:
330-
i = j
329+
return m[0]
331330
else:
332-
tail = path[j:]
333-
path = path[:i] + value
334-
i = len(path)
335-
path += tail
336-
return path
331+
return value
332+
333+
return sub(repl, path)
337334

338335

339336
# Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B.

Lib/test/test_genericpath.py

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,9 @@
77
import sys
88
import unittest
99
import warnings
10-
from test.support import (
11-
is_apple, os_helper, warnings_helper
12-
)
10+
from test import support
11+
from test.support import os_helper
12+
from test.support import warnings_helper
1313
from test.support.script_helper import assert_python_ok
1414
from test.support.os_helper import FakePath
1515

@@ -445,6 +445,19 @@ def check(value, expected):
445445
os.fsencode('$bar%s bar' % nonascii))
446446
check(b'$spam}bar', os.fsencode('%s}bar' % nonascii))
447447

448+
@support.requires_resource('cpu')
449+
def test_expandvars_large(self):
450+
expandvars = self.pathmodule.expandvars
451+
with os_helper.EnvironmentVarGuard() as env:
452+
env.clear()
453+
env["A"] = "B"
454+
n = 100_000
455+
self.assertEqual(expandvars('$A'*n), 'B'*n)
456+
self.assertEqual(expandvars('${A}'*n), 'B'*n)
457+
self.assertEqual(expandvars('$A!'*n), 'B!'*n)
458+
self.assertEqual(expandvars('${A}A'*n), 'BA'*n)
459+
self.assertEqual(expandvars('${'*10*n), '${'*10*n)
460+
448461
def test_abspath(self):
449462
self.assertIn("foo", self.pathmodule.abspath("foo"))
450463
with warnings.catch_warnings():
@@ -502,7 +515,7 @@ def test_nonascii_abspath(self):
502515
# directory (when the bytes name is used).
503516
and sys.platform not in {
504517
"win32", "emscripten", "wasi"
505-
} and not is_apple
518+
} and not support.is_apple
506519
):
507520
name = os_helper.TESTFN_UNDECODABLE
508521
elif os_helper.TESTFN_NONASCII:

Lib/test/test_ntpath.py

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
import sys
77
import unittest
88
import warnings
9-
from test.support import TestFailed, cpython_only, os_helper
9+
from test import support
10+
from test.support import os_helper
1011
from test.support.os_helper import FakePath
1112
from test import test_genericpath
1213
from tempfile import TemporaryFile
@@ -56,7 +57,7 @@ def tester(fn, wantResult):
5657
fn = fn.replace("\\", "\\\\")
5758
gotResult = eval(fn)
5859
if wantResult != gotResult and _norm(wantResult) != _norm(gotResult):
59-
raise TestFailed("%s should return: %s but returned: %s" \
60+
raise support.TestFailed("%s should return: %s but returned: %s" \
6061
%(str(fn), str(wantResult), str(gotResult)))
6162

6263
# then with bytes
@@ -72,7 +73,7 @@ def tester(fn, wantResult):
7273
warnings.simplefilter("ignore", DeprecationWarning)
7374
gotResult = eval(fn)
7475
if _norm(wantResult) != _norm(gotResult):
75-
raise TestFailed("%s should return: %s but returned: %s" \
76+
raise support.TestFailed("%s should return: %s but returned: %s" \
7677
%(str(fn), str(wantResult), repr(gotResult)))
7778

7879

@@ -875,6 +876,19 @@ def check(value, expected):
875876
check('%spam%bar', '%sbar' % nonascii)
876877
check('%{}%bar'.format(nonascii), 'ham%sbar' % nonascii)
877878

879+
@support.requires_resource('cpu')
880+
def test_expandvars_large(self):
881+
expandvars = ntpath.expandvars
882+
with os_helper.EnvironmentVarGuard() as env:
883+
env.clear()
884+
env["A"] = "B"
885+
n = 100_000
886+
self.assertEqual(expandvars('%A%'*n), 'B'*n)
887+
self.assertEqual(expandvars('%A%A'*n), 'BA'*n)
888+
self.assertEqual(expandvars("''"*n + '%%'), "''"*n + '%')
889+
self.assertEqual(expandvars("%%"*n), "%"*n)
890+
self.assertEqual(expandvars("$$"*n), "$"*n)
891+
878892
def test_expanduser(self):
879893
tester('ntpath.expanduser("test")', 'test')
880894

@@ -1292,7 +1306,7 @@ def test_con_device(self):
12921306
self.assertTrue(os.path.exists(r"\\.\CON"))
12931307

12941308
@unittest.skipIf(sys.platform != 'win32', "Fast paths are only for win32")
1295-
@cpython_only
1309+
@support.cpython_only
12961310
def test_fast_paths_in_use(self):
12971311
# There are fast paths of these functions implemented in posixmodule.c.
12981312
# Confirm that they are being used, and not the Python fallbacks in
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Fix quadratic complexity in :func:`os.path.expandvars`.

0 commit comments

Comments
 (0)