Merge pull request github#6665 from smowton/smowton/fix/python-redos-invalid-utf16

ReDoS: fix unpaired surrogate test

Author: yoff

python/ql/test/library-tests/regexparser/redos.py

@@ -266,11 +266,11 @@
 # GOOD
 good27 = re.compile(r'''(thisisagoddamnlongstringforstresstestingthequery|imanotherbutunrelatedstringcomparedtotheotherstring)*-''')
 
-# GOOD
-good28 = re.compile(r'''foo([\uDC66\uDC67]|[\uDC68\uDC69])*foo''')
+# GOOD (but false positive caused by the extractor converting all four unpaired surrogates to \uFFFD)
+good28 = re.compile('''foo([\uDC66\uDC67]|[\uDC68\uDC69])*foo''')
 
-# GOOD
-good29 = re.compile(r'''foo((\uDC66|\uDC67)|(\uDC68|\uDC69))*foo''')
+# GOOD (but false positive caused by the extractor converting all four unpaired surrogates to \uFFFD)
+good29 = re.compile('''foo((\uDC66|\uDC67)|(\uDC68|\uDC69))*foo''')
 
 # NOT GOOD (but cannot currently construct a prefix)
 bad62 = re.compile(r'''a{2,3}(b+)+X''')

python/ql/test/query-tests/Security/CWE-730-ReDoS/ReDoS.expected

@@ -65,6 +65,8 @@
 | redos.py:259:24:259:126 | (.thisisagoddamnlongstringforstresstestingthequery\|\\sthisisagoddamnlongstringforstresstestingthequery)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of ' thisisagoddamnlongstringforstresstestingthequery'. |
 | redos.py:262:24:262:87 | (thisisagoddamnlongstringforstresstestingthequery\|this\\w+query)* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'thisisagoddamnlongstringforstresstestingthequery'. |
 | redos.py:262:78:262:80 | \\w+ | This part of the regular expression may cause exponential backtracking on strings starting with 'this' and containing many repetitions of 'aquerythis'. |
+| redos.py:268:28:268:39 | ([\ufffd\ufffd]\|[\ufffd\ufffd])* | This part of the regular expression may cause exponential backtracking on strings starting with 'foo' and containing many repetitions of '\ufffd'. |
+| redos.py:271:28:271:41 | ((\ufffd\|\ufffd)\|(\ufffd\|\ufffd))* | This part of the regular expression may cause exponential backtracking on strings starting with 'foo' and containing many repetitions of '\ufffd'. |
 | redos.py:274:31:274:32 | b+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'b'. |
 | redos.py:277:48:277:50 | \\s* | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of '"" a='. |
 | redos.py:283:26:283:27 | a+ | This part of the regular expression may cause exponential backtracking on strings containing many repetitions of 'a'. |

python/ql/test/query-tests/Security/CWE-730-ReDoS/redos.py

@@ -264,11 +264,11 @@
 # GOOD
 good27 = re.compile(r'''(thisisagoddamnlongstringforstresstestingthequery|imanotherbutunrelatedstringcomparedtotheotherstring)*-''')
 
-# GOOD
-good28 = re.compile(r'''foo([\uDC66\uDC67]|[\uDC68\uDC69])*foo''')
+# GOOD (but false positive caused by the extractor converting all four unpaired surrogates to \uFFFD)
+good28 = re.compile('''foo([\uDC66\uDC67]|[\uDC68\uDC69])*foo''')
 
-# GOOD
-good29 = re.compile(r'''foo((\uDC66|\uDC67)|(\uDC68|\uDC69))*foo''')
+# GOOD (but false positive caused by the extractor converting all four unpaired surrogates to \uFFFD)
+good29 = re.compile('''foo((\uDC66|\uDC67)|(\uDC68|\uDC69))*foo''')
 
 # NOT GOOD (but cannot currently construct a prefix)
 bad62 = re.compile(r'''a{2,3}(b+)+X''')