[GR-11670] Implement support for 're.split'.

PullRequest: graalpython/194

Author: fangerer

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/GraalPythonEnvVars.java

@@ -57,11 +57,13 @@ private static String discoverHomeFromSource() throws IOException {
             final Path codeDir = codeLocation.getParent();
 
             // executing from jar file in source tree
-            if (codeDir.endsWith(Paths.get("mxbuild", "dists"))) {
-                final Path candidate = codeDir.getParent().getParent().resolve("graalpython");
-                if (isGraalPythonHome(candidate)) {
-                    // Jar source build
-                    return candidate.toFile().getCanonicalPath().toString();
+            for (Path cur = codeDir; cur.getNameCount() >= 2; cur = cur.getParent()) {
+                if (cur.endsWith(Paths.get("mxbuild", "dists"))) {
+                    final Path candidate = cur.getParent().getParent().resolve("graalpython");
+                    if (isGraalPythonHome(candidate)) {
+                        // Jar source build
+                        return candidate.toFile().getCanonicalPath().toString();
+                    }
                 }
             }
 

graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/test/builtin/RegularExpressionTests.java

@@ -58,6 +58,7 @@ def varfunc():
     assert 'x' in var
     assert var['x'] == 10
 
+
 global_var = {}
 
 
@@ -70,3 +71,17 @@ def test_assign_nonlocal_func_return():
 
     assert 'x' in global_var
     assert global_var['x'] == 10
+
+
+def test_destructuring():
+    a, b = (1, 2)
+    assert a == 1 and b == 2
+
+    a, b, c = "\xe0\xdf\xe7"
+    assert a == "à" and b == "ß" and c == "ç"
+
+    a, b, c = "\u0430\u0431\u0432"
+    assert a == 'а' and b == 'б' and c == 'в'
+    # TODO not supported yet
+#     a, b, c = "\U0001d49c\U0001d49e\U0001d4b5"
+#     assert a == '𝒜' and b == '𝒞' and c == '𝒵'

graalpython/com.oracle.graal.python.test/src/tests/test_assign.py

@@ -244,6 +244,86 @@ def test_symbolic_groups(self):
             self.checkPatternError(r'(?(-1))', "bad character in group name '-1'", 3)
             self.assertEqual(re.match(pat, 'xc8yz').span(), (0, 5))
 
+    def test_re_split(self):
+        for string in ":a:b::c", S(":a:b::c"):
+            self.assertTypedEqual(re.split(":", string),
+                                  ['', 'a', 'b', '', 'c'])
+            self.assertTypedEqual(re.split(":+", string),
+                                  ['', 'a', 'b', 'c'])
+            self.assertTypedEqual(re.split("(:+)", string),
+                                  ['', ':', 'a', ':', 'b', '::', 'c'])
+        for string in (b":a:b::c", B(b":a:b::c"), bytearray(b":a:b::c"),
+                       memoryview(b":a:b::c")):
+            self.assertTypedEqual(re.split(b":", string),
+                                  [b'', b'a', b'b', b'', b'c'])
+            self.assertTypedEqual(re.split(b":+", string),
+                                  [b'', b'a', b'b', b'c'])
+            self.assertTypedEqual(re.split(b"(:+)", string),
+                                  [b'', b':', b'a', b':', b'b', b'::', b'c'])
+        # TODO not supported yet
+        # for a, b, c in ("\xe0\xdf\xe7", "\u0430\u0431\u0432", "\U0001d49c\U0001d49e\U0001d4b5"):
+        for a, b, c in ("\xe0\xdf\xe7", "\u0430\u0431\u0432"):
+            string = ":%s:%s::%s" % (a, b, c)
+            self.assertEqual(re.split(":", string), ['', a, b, '', c])
+            self.assertEqual(re.split(":+", string), ['', a, b, c])
+            self.assertEqual(re.split("(:+)", string),
+                             ['', ':', a, ':', b, '::', c])
+
+        self.assertEqual(re.split("(?::+)", ":a:b::c"), ['', 'a', 'b', 'c'])
+        self.assertEqual(re.split("(:)+", ":a:b::c"),
+                         ['', ':', 'a', ':', 'b', ':', 'c'])
+        self.assertEqual(re.split("([b:]+)", ":a:b::c"),
+                         ['', ':', 'a', ':b::', 'c'])
+        self.assertEqual(re.split("(b)|(:+)", ":a:b::c"),
+                         ['', None, ':', 'a', None, ':', '', 'b', None, '',
+                          None, '::', 'c'])
+        self.assertEqual(re.split("(?:b)|(?::+)", ":a:b::c"),
+                         ['', 'a', '', '', 'c'])
+
+        # TODO subtests not support yet
+        # for sep, expected in [
+        #    (':*', ['', 'a', 'b', 'c']),
+        #    ('(?::*)', ['', 'a', 'b', 'c']),
+        #    ('(:*)', ['', ':', 'a', ':', 'b', '::', 'c']),
+        #    ('(:)*', ['', ':', 'a', ':', 'b', ':', 'c']),
+        # ]:
+        #    with self.subTest(sep=sep), self.assertWarns(FutureWarning):
+        #       self.assertTypedEqual(re.split(sep, ':a:b::c'), expected)
+        # for sep, expected in [
+        #    ('', [':a:b::c']),
+        #    (r'\b', [':a:b::c']),
+        #    (r'(?=:)', [':a:b::c']),
+        #    (r'(?<=:)', [':a:b::c']),
+        # ]:
+        #    with self.subTest(sep=sep), self.assertRaises(ValueError):
+        #        self.assertTypedEqual(re.split(sep, ':a:b::c'), expected)
+
+    def test_re_findall(self):
+        self.assertEqual(re.findall(":+", "abc"), [])
+        for string in "a:b::c:::d", S("a:b::c:::d"):
+            self.assertTypedEqual(re.findall(":+", string),
+                                  [":", "::", ":::"])
+            self.assertTypedEqual(re.findall("(:+)", string),
+                                  [":", "::", ":::"])
+            self.assertTypedEqual(re.findall("(:)(:*)", string),
+                                  [(":", ""), (":", ":"), (":", "::")])
+        for string in (b"a:b::c:::d", B(b"a:b::c:::d"), bytearray(b"a:b::c:::d"),
+                       memoryview(b"a:b::c:::d")):
+            self.assertTypedEqual(re.findall(b":+", string),
+                                  [b":", b"::", b":::"])
+            self.assertTypedEqual(re.findall(b"(:+)", string),
+                                  [b":", b"::", b":::"])
+            self.assertTypedEqual(re.findall(b"(:)(:*)", string),
+                                  [(b":", b""), (b":", b":"), (b":", b"::")])
+        for x in ("\xe0", "\u0430", "\U0001d49c"):
+            xx = x * 2
+            xxx = x * 3
+            string = "a%sb%sc%sd" % (x, xx, xxx)
+            self.assertEqual(re.findall("%s+" % x, string), [x, xx, xxx])
+            self.assertEqual(re.findall("(%s+)" % x, string), [x, xx, xxx])
+            self.assertEqual(re.findall("(%s)(%s*)" % (x, x), string),
+                             [(x, ""), (x, x), (x, xx)])
+
     def test_ignore_case_set(self):
         self.assertTrue(re.match(r'[19A]', 'A', re.I))
         self.assertTrue(re.match(r'[19a]', 'a', re.I))

graalpython/com.oracle.graal.python.test/src/tests/test_re.py

@@ -111,7 +111,7 @@ protected List<com.oracle.truffle.api.dsl.NodeFactory<? extends PythonBuiltinBas
     public abstract static class StrNode extends PythonUnaryBuiltinNode {
         @Specialization
         public Object str(PString self) {
-            return self;
+            return self.getValue();
         }
 
         @Specialization

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/str/StringBuiltins.java

@@ -730,6 +730,26 @@ private static String unescapeJavaString(String st) {
                         sb.append(Character.toChars(code));
                         i += 5;
                         continue;
+                    // Hex Unicode: U????????
+                    case 'U':
+                        if (i >= st.length() - 9) {
+                            ch = 'U';
+                            break;
+                        }
+                        code = Integer.parseInt(st.substring(i + 2, i + 10), 16);
+                        sb.append(Character.toChars(code));
+                        i += 9;
+                        continue;
+                    // Hex Unicode: x??
+                    case 'x':
+                        if (i >= st.length() - 3) {
+                            ch = 'u';
+                            break;
+                        }
+                        int hexCode = Integer.parseInt("" + st.charAt(i + 2) + st.charAt(i + 3), 16);
+                        sb.append(Character.toChars(hexCode));
+                        i += 3;
+                        continue;
                 }
                 i++;
             }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/parser/PythonTreeTranslator.java

@@ -255,10 +255,13 @@ def findall(self, string, pos=0, endpos=-1):
                 result = tregex_call_safe(pattern.exec, string, pos)
                 if not result.isMatch:
                     break
-                elif self.num_groups == 0:
-                    matchlist.append("")
-                elif self.num_groups == 1:
-                    matchlist.append(string[result.start[1]:result.end[1]])
+                elif result.groupCount == 0:
+                    assert False, "inconsistent regex result"
+                    matchlist.append('')
+                elif result.groupCount == 1:
+                    matchlist.append(str(string[result.start[0]:result.end[0]]))
+                elif result.groupCount == 2:
+                    matchlist.append(str(string[result.start[1]:result.end[1]]))
                 else:
                     matchlist.append(SRE_Match(self, pos, endpos, result).groups())
                 no_progress = (result.start[0] == result.end[0])
@@ -355,6 +358,36 @@ def sub(self, repl, string, count=0):
             return self.__compile_cpython_sre().sub(repl, string, count)
 
 
+    def split(self, string, maxsplit=0):
+        n = 0
+        try:
+            pattern = self.__tregex_compile(self.pattern)
+            result = []
+            pos = 0
+            progress = True
+            while (maxsplit == 0 or n < maxsplit) and pos <= len(string) and progress:
+                match_result = tregex_call_safe(pattern.exec, string, pos)
+                if not match_result.isMatch:
+                    break
+                n += 1
+                start = match_result.start[0]
+                end = match_result.end[0]
+                result.append(str(string[pos:start]))
+                # add all group strings
+                for i in range(match_result.groupCount-1):
+                    groupStart = match_result.start[i + 1]
+                    if groupStart >= 0:
+                        result.append(str(string[groupStart:match_result.end[i+1]]))
+                    else:
+                        result.append(None)
+                pos = end
+                progress = (start != end)
+            result.append(str(string[pos:]))
+            return result
+        except BaseException:
+            return self.__compile_cpython_sre().split(string, maxsplit)
+
+
 compile = SRE_Pattern