Improved statement bind variable placeholder parser performance and

accuracy (now supports q-strings and JSON constant syntax as well as
embedded quotes as reported in #208).

Author: anthony-tuininga

doc/src/release_notes.rst

@@ -13,6 +13,12 @@ oracledb 2.0.0 (TBD)
 Thin Mode Changes
 +++++++++++++++++
 
+#)  Improved statement bind variable placeholder parser performance, handle
+    statements which use the `Alternative Quoting Mechanism
+    <https://www.oracle.com/pls/topic/lookup?ctx=dblatest&id=GUID-1824CBAA-6E16-4921-B2A6-112FB02248DA>`__
+    ('Q' strings), and fix some issues identifying bind variable placeholders
+    in embedded quotes and in JSON syntax.
+
 Thick Mode Changes
 ++++++++++++++++++
 

src/oracledb/impl/thin/statement.pyx

@@ -30,27 +30,6 @@
 # may be bound to those statements (embedded in thin_impl.pyx).
 #------------------------------------------------------------------------------
 
-# Rules for named binds:
-# 1. Quoted and non-quoted bind names are allowed.
-# 2. Quoted binds can contain any characters.
-# 3. Non-quoted binds must begin with an alphabet character.
-# 4. Non-quoted binds can only contain alphanumeric characters, the underscore,
-#    the dollar sign and the pound sign.
-# 5. Non-quoted binds cannot be Oracle Database Reserved Names (Server handles
-#    this case and returns an appropriate error)
-BIND_PATTERN = r':\s*((?:".*?")|(?:[^\W\d_][\w\$#]*)|\d+)'
-
-# pattern used for detecting a DML returning clause; bind variables in the
-# SQL prior to the INTO keyword are input variables; bind varibles in the SQL
-# after the INTO keyword are output variables
-DML_RETURNING_PATTERN = r'(?si)(?<=\bRETURNING\b)(.*?)(?=\bINTO\b)'
-
-# patterns for identifying comments and quoted strings
-SINGLE_LINE_COMMENT_PATTERN = r'--.*'
-MULTI_LINE_COMMENT_PATTERN = r'(?s)/\*.*?\*/'
-CONSTANT_STRING_PATTERN = r"(?s)'.*?'"
-QUOTED_NAME_PATTERN = r'(:\s*)?(".*?")'
-
 cdef class BindInfo:
 
     cdef:
@@ -75,6 +54,244 @@ cdef class BindInfo:
         return BindInfo(self._bind_name, self._is_return_bind)
 
 
+cdef class Parser:
+
+    cdef:
+        bint returning_keyword_found
+        ssize_t pos, max_pos
+        void* sql_data
+        int sql_kind
+
+    cdef int _parse_bind_name(self, Statement stmt) except -1:
+        """
+        Bind variables are identified as follows:
+        - Quoted and non-quoted bind names are allowed.
+        - Quoted bind names can contain any characters.
+        - Non-quoted bind names must begin with an alphabetic character.
+        - Non-quoted bind names can only contain alphanumeric characters, the
+          underscore, the dollar sign and the pound sign.
+        - Non-quoted bind names cannot be Oracle Database Reserved Names (this
+          is left to the server to detct and return an appropriate error).
+        """
+        cdef:
+            bint quoted_name = False, in_bind = False, digits_only = False
+            ssize_t start_pos = 0, pos = self.pos + 1
+            str bind_name
+            Py_UCS4 ch
+        while pos <= self.max_pos:
+            ch = cpython.PyUnicode_READ(self.sql_kind, self.sql_data, pos)
+            if not in_bind:
+                if cpython.Py_UNICODE_ISSPACE(ch):
+                    pos += 1
+                    continue
+                elif ch == '"':
+                    quoted_name = True
+                elif cpython.Py_UNICODE_ISDIGIT(ch):
+                    digits_only = True
+                elif not cpython.Py_UNICODE_ISALPHA(ch):
+                    break
+                in_bind = True
+                start_pos = pos
+            elif digits_only and not cpython.Py_UNICODE_ISDIGIT(ch):
+                self.pos = pos - 1
+                break
+            elif quoted_name and ch == '"':
+                self.pos = pos
+                break
+            elif not digits_only and not quoted_name \
+                    and not cpython.Py_UNICODE_ISALNUM(ch) \
+                    and ch not in ('_', '$', '#'):
+                self.pos = pos - 1
+                break
+            pos += 1
+        if in_bind:
+            if quoted_name:
+                bind_name = stmt._sql[start_pos + 1:pos]
+            elif digits_only:
+                bind_name = stmt._sql[start_pos:pos]
+            else:
+                bind_name = stmt._sql[start_pos:pos].upper()
+            stmt._add_bind(bind_name)
+
+    cdef int _parse_multiple_line_comment(self) except -1:
+        """
+        Multiple line comments consist of the characters /* followed by all
+        characters up until */. This method is called when the first slash is
+        detected and checks for the subsequent asterisk. If found, the comment
+        is traversed and the current position is updaqted; otherwise, the
+        current position is left untouched.
+        """
+        cdef:
+            bint in_comment = False, exiting_comment = False
+            ssize_t pos = self.pos + 1
+            Py_UCS4 ch
+        while pos <= self.max_pos:
+            ch = cpython.PyUnicode_READ(self.sql_kind, self.sql_data, pos)
+            if not in_comment:
+                if ch != '*':
+                    break
+                in_comment = True
+            elif not exiting_comment and ch == '*':
+                exiting_comment = True
+            elif exiting_comment:
+                if ch == '/':
+                    self.pos = pos
+                    break
+                exiting_comment = False
+            pos += 1
+
+    cdef int _parse_qstring(self) except -1:
+        """
+        Parses a q-string which consists of the characters "q" and a single
+        quote followed by a start separator, any text that does not contain the
+        end seprator and the end separator and ending quote. The following are
+        examples that demonstrate this:
+            - q'[...]'
+            - q'{...}'
+            - q'<...>'
+            - q'(...)'
+            - q'?...?' (where ? is any character)
+        """
+        cdef:
+            bint exiting_qstring = False, in_qstring = False
+            Py_UCS4 ch, sep = 0
+        self.pos += 1
+        while self.pos <= self.max_pos:
+            ch = cpython.PyUnicode_READ(self.sql_kind, self.sql_data, self.pos)
+            if not in_qstring:
+                if ch == '[':
+                    sep = ']'
+                elif ch == '{':
+                    sep = '}'
+                elif ch == '<':
+                    sep = '>'
+                elif ch == '(':
+                    sep = ')'
+                else:
+                    sep = ch
+                in_qstring = True
+            elif not exiting_qstring and ch == sep:
+                exiting_qstring = True
+            elif exiting_qstring:
+                if ch == "'":
+                    break
+                elif ch != sep:
+                    exiting_qstring = False
+            self.pos += 1
+
+    cdef int _parse_quoted_string(self, Py_UCS4 sep) except -1:
+        """
+        Parses a quoted string with the given separator. All characters until
+        the separate is detected are discarded.
+        """
+        cdef Py_UCS4 ch
+        self.pos += 1
+        while self.pos <= self.max_pos:
+            ch = cpython.PyUnicode_READ(self.sql_kind, self.sql_data, self.pos)
+            if ch == sep:
+                break
+            self.pos += 1
+
+    cdef int _parse_single_line_comment(self) except -1:
+        """
+        Single line comments consist of two dashes and all characters up to the
+        next line break. This method is called when the first dash is detected
+        and checks for the subsequent dash. If found, the single line comment
+        is traversed and the current position is updated; otherwise, the
+        current position is left untouched.
+        """
+        cdef:
+            ssize_t pos = self.pos + 1
+            bint in_comment = False
+            Py_UCS4 ch
+        while pos <= self.max_pos:
+            ch = cpython.PyUnicode_READ(self.sql_kind, self.sql_data, pos)
+            if not in_comment:
+                if ch != '-':
+                    break
+                in_comment = True
+            elif cpython.Py_UNICODE_ISLINEBREAK(ch):
+                self.pos = pos
+                break
+            pos += 1
+
+    cdef int parse(self, Statement stmt) except -1:
+        """
+        Parses the SQL stored in the statement in order to determine the
+        keyword that identifies the type of SQL being executed as well as a
+        list of bind variable names. A check is also made for DML returning
+        statements since the bind variables following the "INTO" keyword are
+        treated differently from other bind variables.
+        """
+        cdef:
+            bint initial_keyword_found = False, last_was_string = False
+            Py_UCS4 ch, last_ch = 0, alpha_start_ch = 0
+            ssize_t alpha_start_pos = 0, alpha_len
+            bint last_was_alpha = False, is_alpha
+            str keyword
+
+        # initialization
+        self.pos = 0
+        self.max_pos = cpython.PyUnicode_GET_LENGTH(stmt._sql) - 1
+        self.sql_kind = cpython.PyUnicode_KIND(stmt._sql)
+        self.sql_data = cpython.PyUnicode_DATA(stmt._sql)
+
+        # scan all characters in the string
+        while self.pos <= self.max_pos:
+            ch = cpython.PyUnicode_READ(self.sql_kind, self.sql_data, self.pos)
+
+            # look for certain keywords (initial keyword and the ones for
+            # detecting DML returning statements
+            is_alpha = cpython.Py_UNICODE_ISALPHA(ch)
+            if is_alpha and not last_was_alpha:
+                alpha_start_pos = self.pos
+                alpha_start_ch = ch
+            elif not is_alpha and last_was_alpha:
+                alpha_len = self.pos - alpha_start_pos
+                if not initial_keyword_found:
+                    keyword = stmt._sql[alpha_start_pos:self.pos].upper()
+                    stmt._determine_statement_type(keyword)
+                    if stmt._is_ddl:
+                        break
+                    initial_keyword_found = True
+                elif stmt._is_dml and not self.returning_keyword_found \
+                        and alpha_len == 9 and alpha_start_ch in ('r', 'R'):
+                    keyword = stmt._sql[alpha_start_pos:self.pos].upper()
+                    if keyword == "RETURNING":
+                        self.returning_keyword_found = True
+                elif self.returning_keyword_found and alpha_len == 4 \
+                        and alpha_start_ch in ('i', 'I'):
+                    keyword = stmt._sql[alpha_start_pos:self.pos].upper()
+                    if keyword == "INTO":
+                        stmt._is_returning = True
+
+            # need to keep track of whether the last token parsed was a string
+            # (excluding whitespace) as if the last token parsed was a string
+            # a following colon is not a bind variable but a part of the JSON
+            # constant syntax
+            if ch == "'":
+                last_was_string = True
+                if last_ch in ('q', 'Q'):
+                    self._parse_qstring()
+                else:
+                    self._parse_quoted_string(ch)
+            elif not cpython.Py_UNICODE_ISSPACE(ch):
+                if ch == '-':
+                    self._parse_single_line_comment()
+                elif ch == '/':
+                    self._parse_multiple_line_comment()
+                elif ch == '"':
+                    self._parse_quoted_string(ch)
+                elif ch == ':' and not last_was_string:
+                    self._parse_bind_name(stmt)
+                last_was_string = False
+
+            # advance to next character and track previous character
+            self.pos += 1
+            last_was_alpha = is_alpha
+            last_ch = ch
+
+
 cdef class Statement:
 
     cdef:
@@ -126,94 +343,53 @@ cdef class Statement:
         copied_statement._return_to_cache = False
         return copied_statement
 
-    cdef int _add_binds(self, str sql, bint is_return_bind) except -1:
+    cdef int _add_bind(self, str name) except -1:
         """
         Add bind information to the statement by examining the passed SQL for
         bind variable names.
         """
-        cdef:
-            BindInfo info
-            str name
-        for name in re.findall(BIND_PATTERN, sql):
-            if name.startswith('"') and name.endswith('"'):
-                name = name[1:-1]
-            else:
-                name = name.upper()
-            if self._is_plsql and name in self._bind_info_dict:
-                continue
-            info = BindInfo(name, is_return_bind)
+        cdef BindInfo info
+        if not self._is_plsql or name not in self._bind_info_dict:
+            info = BindInfo(name, self._is_returning)
             self._bind_info_list.append(info)
             if info._bind_name in self._bind_info_dict:
                 self._bind_info_dict[info._bind_name].append(info)
             else:
                 self._bind_info_dict[info._bind_name] = [info]
 
-    cdef _determine_statement_type(self, str sql):
+    cdef _determine_statement_type(self, str sql_keyword):
         """
         Determine the type of the SQL statement by examining the first keyword
         found in the statement.
         """
-        tokens = sql.strip().lstrip("(")[:10].split()
-        if tokens:
-            sql_keyword = tokens[0].upper()
-            if sql_keyword in ("DECLARE", "BEGIN", "CALL"):
-                self._is_plsql = True
-            elif sql_keyword in ("SELECT", "WITH"):
-                self._is_query = True
-            elif sql_keyword in ("INSERT", "UPDATE", "DELETE", "MERGE"):
-                self._is_dml = True
-            elif sql_keyword in ("CREATE", "ALTER", "DROP", "TRUNCATE"):
-                self._is_ddl = True
+        if sql_keyword in ("DECLARE", "BEGIN", "CALL"):
+            self._is_plsql = True
+        elif sql_keyword in ("SELECT", "WITH"):
+            self._is_query = True
+        elif sql_keyword in ("INSERT", "UPDATE", "DELETE", "MERGE"):
+            self._is_dml = True
+        elif sql_keyword in ("CREATE", "ALTER", "DROP", "GRANT", "REVOKE",
+                             "ANALYZE", "AUDIT", "COMMENT", "TRUNCATE"):
+            self._is_ddl = True
 
     cdef int _prepare(self, str sql) except -1:
         """
         Prepare the SQL for execution by determining the list of bind names
         that are found within it. The length of the SQL text is also calculated
-        at this time. If the character sets of the client and server are
-        identical, the length is calculated in bytes; otherwise, the length is
-        calculated in characters.
+        at this time.
         """
-        cdef:
-            str input_sql, returning_sql = None
-            object match
+        cdef Parser parser = Parser.__new__(Parser)
 
         # retain normalized SQL (as string and bytes) as well as the length
         self._sql = sql
         self._sql_bytes = self._sql.encode()
         self._sql_length = <uint32_t> len(self._sql_bytes)
 
-        # create empty list (bind by position) and dict (bind by name)
+        # parse SQL and populate bind variable list (bind by position) and dict
+        # (bind by name)
         self._bind_info_dict = collections.OrderedDict()
         self._bind_info_list = []
-
-        # Strip single/multiline comments and replace constant strings and
-        # quoted names with single characters in order to facilitate detection
-        # of bind variables; note that bind variables can be quoted so a check
-        # must be made to ensure that a quoted string doesn't refer to a bind
-        # variable first before it can be replaced
-        sql = re.sub(MULTI_LINE_COMMENT_PATTERN, "", sql)
-        sql = re.sub(SINGLE_LINE_COMMENT_PATTERN, "", sql)
-        sql = re.sub(CONSTANT_STRING_PATTERN, "S", sql)
-        sql = re.sub(QUOTED_NAME_PATTERN,
-                    lambda m: m.group(0) if sql[m.start(0)] == ":" else "Q",
-                    sql)
-
-        # determine statement type
-        self._determine_statement_type(sql)
-
-        # bind variables can only be found in queries, DML and PL/SQL
-        if self._is_query or self._is_dml or self._is_plsql:
-            input_sql = sql
-            if self._is_dml:
-                match = re.search(DML_RETURNING_PATTERN, sql)
-                if match is not None:
-                    pos = match.end()
-                    input_sql = sql[:pos]
-                    returning_sql = sql[pos + 4:]
-            self._add_binds(input_sql, is_return_bind=False)
-            if returning_sql is not None:
-                self._is_returning = True
-                self._add_binds(returning_sql, is_return_bind=True)
+        parser.parse(self)
 
     cdef int _set_var(self, BindInfo bind_info, ThinVarImpl var_impl,
                       ThinCursorImpl cursor_impl) except -1: