[SPARK-42342][PYTHON][CONNECT] Introduce base hierarchy to exceptions

### What changes were proposed in this pull request?

Introduces base hierarchy to exceptions.

As a common hierarchy for users, base exception classes are subclasses of `PySparkException`.
The concrete classes for both PySpark and Spark Connect inherits the base classes that should not be exposed to users.

### Why are the changes needed?

Currently exception class hierarchy is separated between PySpark and Spark Connect.

If users want to check the exception type, they need to switch the error classes based on whether they are running on PySpark or Spark Connect, but it's not ideal.

### Does this PR introduce _any_ user-facing change?

No. Users still can use the existing exception classes to check the exception type.

### How was this patch tested?

Updated tests.

Closes apache#39882 from ueshin/issues/SPARK-42342/exceptions.

Authored-by: Takuya UESHIN <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
(cherry picked from commit bd34b16)
Signed-off-by: Hyukjin Kwon <[email protected]>

python/docs/source/reference/pyspark.errors.rst

@@ -30,19 +30,14 @@ Classes
 
     PySparkException
     AnalysisException
+    TempTableAlreadyExistsException
     ParseException
     IllegalArgumentException
     StreamingQueryException
     QueryExecutionException
     PythonException
     UnknownException
     SparkUpgradeException
-    SparkConnectAnalysisException
-    SparkConnectException
-    SparkConnectGrpcException
-    SparkConnectParseException
-    SparkConnectTempTableAlreadyExistsException
-    SparkConnectIllegalArgumentException
 
 
 Methods

python/pyspark/errors/__init__.py

@@ -18,9 +18,10 @@
 """
 PySpark exceptions.
 """
-from pyspark.errors.exceptions import (  # noqa: F401
+from pyspark.errors.exceptions.base import (  # noqa: F401
     PySparkException,
     AnalysisException,
+    TempTableAlreadyExistsException,
     ParseException,
     IllegalArgumentException,
     StreamingQueryException,
@@ -30,18 +31,13 @@
     SparkUpgradeException,
     PySparkTypeError,
     PySparkValueError,
-    SparkConnectException,
-    SparkConnectGrpcException,
-    SparkConnectAnalysisException,
-    SparkConnectParseException,
-    SparkConnectTempTableAlreadyExistsException,
-    SparkConnectIllegalArgumentException,
 )
 
 
 __all__ = [
     "PySparkException",
     "AnalysisException",
+    "TempTableAlreadyExistsException",
     "ParseException",
     "IllegalArgumentException",
     "StreamingQueryException",
@@ -51,10 +47,4 @@
     "SparkUpgradeException",
     "PySparkTypeError",
     "PySparkValueError",
-    "SparkConnectException",
-    "SparkConnectGrpcException",
-    "SparkConnectAnalysisException",
-    "SparkConnectParseException",
-    "SparkConnectTempTableAlreadyExistsException",
-    "SparkConnectIllegalArgumentException",
 ]

python/pyspark/errors/exceptions/__init__.py

@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#

python/pyspark/errors/exceptions/base.py

@@ -0,0 +1,162 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from typing import Dict, Optional, cast
+
+from pyspark.errors.utils import ErrorClassesReader
+
+
+class PySparkException(Exception):
+    """
+    Base Exception for handling errors generated from PySpark.
+    """
+
+    def __init__(
+        self,
+        message: Optional[str] = None,
+        error_class: Optional[str] = None,
+        message_parameters: Optional[Dict[str, str]] = None,
+    ):
+        # `message` vs `error_class` & `message_parameters` are mutually exclusive.
+        assert (message is not None and (error_class is None and message_parameters is None)) or (
+            message is None and (error_class is not None and message_parameters is not None)
+        )
+
+        self.error_reader = ErrorClassesReader()
+
+        if message is None:
+            self.message = self.error_reader.get_error_message(
+                cast(str, error_class), cast(Dict[str, str], message_parameters)
+            )
+        else:
+            self.message = message
+
+        self.error_class = error_class
+        self.message_parameters = message_parameters
+
+    def getErrorClass(self) -> Optional[str]:
+        """
+        Returns an error class as a string.
+
+        .. versionadded:: 3.4.0
+
+        See Also
+        --------
+        :meth:`PySparkException.getMessageParameters`
+        :meth:`PySparkException.getSqlState`
+        """
+        return self.error_class
+
+    def getMessageParameters(self) -> Optional[Dict[str, str]]:
+        """
+        Returns a message parameters as a dictionary.
+
+        .. versionadded:: 3.4.0
+
+        See Also
+        --------
+        :meth:`PySparkException.getErrorClass`
+        :meth:`PySparkException.getSqlState`
+        """
+        return self.message_parameters
+
+    def getSqlState(self) -> None:
+        """
+        Returns an SQLSTATE as a string.
+
+        Errors generated in Python have no SQLSTATE, so it always returns None.
+
+        .. versionadded:: 3.4.0
+
+        See Also
+        --------
+        :meth:`PySparkException.getErrorClass`
+        :meth:`PySparkException.getMessageParameters`
+        """
+        return None
+
+    def __str__(self) -> str:
+        if self.getErrorClass() is not None:
+            return f"[{self.getErrorClass()}] {self.message}"
+        else:
+            return self.message
+
+
+class AnalysisException(PySparkException):
+    """
+    Failed to analyze a SQL query plan.
+    """
+
+
+class TempTableAlreadyExistsException(AnalysisException):
+    """
+    Failed to create temp view since it is already exists.
+    """
+
+
+class ParseException(PySparkException):
+    """
+    Failed to parse a SQL command.
+    """
+
+
+class IllegalArgumentException(PySparkException):
+    """
+    Passed an illegal or inappropriate argument.
+    """
+
+
+class StreamingQueryException(PySparkException):
+    """
+    Exception that stopped a :class:`StreamingQuery`.
+    """
+
+
+class QueryExecutionException(PySparkException):
+    """
+    Failed to execute a query.
+    """
+
+
+class PythonException(PySparkException):
+    """
+    Exceptions thrown from Python workers.
+    """
+
+
+class UnknownException(PySparkException):
+    """
+    None of the above exceptions.
+    """
+
+
+class SparkUpgradeException(PySparkException):
+    """
+    Exception thrown because of Spark upgrade.
+    """
+
+
+class PySparkValueError(PySparkException, ValueError):
+    """
+    Wrapper class for ValueError to support error classes.
+    """
+
+
+class PySparkTypeError(PySparkException, TypeError):
+    """
+    Wrapper class for TypeError to support error classes.
+    """