Merge with main branch

Author: vinjai

.github/workflows/python-release.yml

@@ -63,7 +63,7 @@ jobs:
         if: startsWith(matrix.os, 'ubuntu')
 
       - name: Build wheels
-        uses: pypa/[email protected].1
+        uses: pypa/[email protected].3
         with:
           output-dir: wheelhouse
           config-file: "pyproject.toml"

mkdocs/docs/configuration.md

@@ -82,10 +82,10 @@ export LAST_COMMIT_ID=$(git rev-list ${GIT_TAG} 2> /dev/null | head -n 1)
 ```
 
 The `-s` option will sign the commit. If you don't have a key yet, you can find the instructions [here](http://www.apache.org/dev/openpgp.html#key-gen-generate-key). To install gpg on a M1 based Mac, a couple of additional steps are required: <https://gist.github.com/phortuin/cf24b1cca3258720c71ad42977e1ba57>.
-If you have not published your GPG key in [KEYS](https://dist.apache.org/repos/dist/dev/iceberg/KEYS) yet, you must publish it before sending the vote email by doing:
+If you have not published your GPG key in [KEYS](https://downloads.apache.org/iceberg/KEYS) yet, you must publish it before sending the vote email by doing:
 
 ```bash
-svn co https://dist.apache.org/repos/dist/dev/iceberg icebergsvn
+svn co https://dist.apache.org/repos/dist/release/iceberg icebergsvn
 cd icebergsvn
 echo "" >> KEYS # append a newline
 gpg --list-sigs <YOUR KEY ID HERE> >> KEYS # append signatures
@@ -166,7 +166,7 @@ The release tarball, signature, and checksums are here:
 
 You can find the KEYS file here:
 
-* https://dist.apache.org/repos/dist/dev/iceberg/KEYS
+* https://downloads.apache.org/iceberg/KEYS
 
 Convenience binary artifacts are staged on pypi:
 

mkdocs/docs/how-to-release.md

@@ -40,7 +40,7 @@ After downloading the source tarball, signature, checksum, and KEYS file, here a
 First, import the keys.
 
 ```sh
-curl https://dist.apache.org/repos/dist/dev/iceberg/KEYS -o KEYS
+curl https://downloads.apache.org/iceberg/KEYS -o KEYS
 gpg --import KEYS
 ```
 

mkdocs/docs/verify-release.md

@@ -16,13 +16,13 @@
 # under the License.
 
 mkdocs==1.6.1
-griffe==1.3.1
+griffe==1.5.1
 jinja2==3.1.4
-mkdocstrings==0.26.1
-mkdocstrings-python==1.11.1
+mkdocstrings==0.26.2
+mkdocstrings-python==1.12.2
 mkdocs-literate-nav==0.6.1
 mkdocs-autorefs==1.2.0
 mkdocs-gen-files==0.5.0
-mkdocs-material==9.5.36
+mkdocs-material==9.5.43
 mkdocs-material-extensions==1.3.1
 mkdocs-section-index==0.3.9

mkdocs/requirements.txt

@@ -15,4 +15,4 @@
 # specific language governing permissions and limitations
 # under the License.
 
-__version__ = "0.7.1"
+__version__ = "0.8.0"

poetry.lock

@@ -106,20 +106,7 @@
     re.X,
 )
 
-DEPRECATED_PROFILE_NAME = "profile_name"
-DEPRECATED_REGION = "region_name"
 DEPRECATED_BOTOCORE_SESSION = "botocore_session"
-DEPRECATED_ACCESS_KEY_ID = "aws_access_key_id"
-DEPRECATED_SECRET_ACCESS_KEY = "aws_secret_access_key"
-DEPRECATED_SESSION_TOKEN = "aws_session_token"
-DEPRECATED_PROPERTY_NAMES = {
-    DEPRECATED_PROFILE_NAME,
-    DEPRECATED_REGION,
-    DEPRECATED_BOTOCORE_SESSION,
-    DEPRECATED_ACCESS_KEY_ID,
-    DEPRECATED_SECRET_ACCESS_KEY,
-    DEPRECATED_SESSION_TOKEN,
-}
 
 
 class CatalogType(Enum):
@@ -794,13 +781,12 @@ class MetastoreCatalog(Catalog, ABC):
     def __init__(self, name: str, **properties: str):
         super().__init__(name, **properties)
 
-        for property_name in DEPRECATED_PROPERTY_NAMES:
-            if self.properties.get(property_name):
-                deprecation_message(
-                    deprecated_in="0.7.0",
-                    removed_in="0.8.0",
-                    help_message=f"The property {property_name} is deprecated. Please use properties that start with client., glue., and dynamo. instead",
-                )
+        if self.properties.get(DEPRECATED_BOTOCORE_SESSION):
+            deprecation_message(
+                deprecated_in="0.8.0",
+                removed_in="0.9.0",
+                help_message=f"The property {DEPRECATED_BOTOCORE_SESSION} is deprecated and will be removed.",
+            )
 
     def create_table_transaction(
         self,
@@ -1011,4 +997,4 @@ def _empty_table_metadata() -> TableMetadata:
         Returns:
             TableMetadata: An empty TableMetadata instance.
         """
-        return TableMetadataV1(location="", last_column_id=-1, schema=Schema())
+        return TableMetadataV1.model_construct(last_column_id=-1, schema=Schema())

pyiceberg/__init__.py

@@ -30,12 +30,7 @@
 import boto3
 
 from pyiceberg.catalog import (
-    DEPRECATED_ACCESS_KEY_ID,
     DEPRECATED_BOTOCORE_SESSION,
-    DEPRECATED_PROFILE_NAME,
-    DEPRECATED_REGION,
-    DEPRECATED_SECRET_ACCESS_KEY,
-    DEPRECATED_SESSION_TOKEN,
     ICEBERG,
     METADATA_LOCATION,
     PREVIOUS_METADATA_LOCATION,
@@ -102,18 +97,12 @@ def __init__(self, name: str, **properties: str):
         super().__init__(name, **properties)
 
         session = boto3.Session(
-            profile_name=get_first_property_value(properties, DYNAMODB_PROFILE_NAME, DEPRECATED_PROFILE_NAME),
-            region_name=get_first_property_value(properties, DYNAMODB_REGION, AWS_REGION, DEPRECATED_REGION),
+            profile_name=properties.get(DYNAMODB_PROFILE_NAME),
+            region_name=get_first_property_value(properties, DYNAMODB_REGION, AWS_REGION),
             botocore_session=properties.get(DEPRECATED_BOTOCORE_SESSION),
-            aws_access_key_id=get_first_property_value(
-                properties, DYNAMODB_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID, DEPRECATED_ACCESS_KEY_ID
-            ),
-            aws_secret_access_key=get_first_property_value(
-                properties, DYNAMODB_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY, DEPRECATED_SECRET_ACCESS_KEY
-            ),
-            aws_session_token=get_first_property_value(
-                properties, DYNAMODB_SESSION_TOKEN, AWS_SESSION_TOKEN, DEPRECATED_SESSION_TOKEN
-            ),
+            aws_access_key_id=get_first_property_value(properties, DYNAMODB_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=get_first_property_value(properties, DYNAMODB_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY),
+            aws_session_token=get_first_property_value(properties, DYNAMODB_SESSION_TOKEN, AWS_SESSION_TOKEN),
         )
         self.dynamodb = session.client(DYNAMODB_CLIENT)
         self.dynamodb_table_name = self.properties.get(DYNAMODB_TABLE_NAME, DYNAMODB_TABLE_NAME_DEFAULT)
@@ -397,7 +386,7 @@ def drop_namespace(self, namespace: Union[str, Identifier]) -> None:
             raise NoSuchNamespaceError(f"Database does not exist: {database_name}") from e
 
     def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]:
-        """List tables under the given namespace in the catalog (including non-Iceberg tables).
+        """List Iceberg tables under the given namespace in the catalog.
 
         Args:
             namespace (str | Identifier): Namespace identifier to search.

pyiceberg/catalog/__init__.py

@@ -40,12 +40,7 @@
 )
 
 from pyiceberg.catalog import (
-    DEPRECATED_ACCESS_KEY_ID,
     DEPRECATED_BOTOCORE_SESSION,
-    DEPRECATED_PROFILE_NAME,
-    DEPRECATED_REGION,
-    DEPRECATED_SECRET_ACCESS_KEY,
-    DEPRECATED_SESSION_TOKEN,
     EXTERNAL_TABLE,
     ICEBERG,
     LOCATION,
@@ -303,18 +298,12 @@ def __init__(self, name: str, **properties: Any):
         super().__init__(name, **properties)
 
         session = boto3.Session(
-            profile_name=get_first_property_value(properties, GLUE_PROFILE_NAME, DEPRECATED_PROFILE_NAME),
-            region_name=get_first_property_value(properties, GLUE_REGION, AWS_REGION, DEPRECATED_REGION),
+            profile_name=properties.get(GLUE_PROFILE_NAME),
+            region_name=get_first_property_value(properties, GLUE_REGION, AWS_REGION),
             botocore_session=properties.get(DEPRECATED_BOTOCORE_SESSION),
-            aws_access_key_id=get_first_property_value(
-                properties, GLUE_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID, DEPRECATED_ACCESS_KEY_ID
-            ),
-            aws_secret_access_key=get_first_property_value(
-                properties, GLUE_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY, DEPRECATED_SECRET_ACCESS_KEY
-            ),
-            aws_session_token=get_first_property_value(
-                properties, GLUE_SESSION_TOKEN, AWS_SESSION_TOKEN, DEPRECATED_SESSION_TOKEN
-            ),
+            aws_access_key_id=get_first_property_value(properties, GLUE_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID),
+            aws_secret_access_key=get_first_property_value(properties, GLUE_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY),
+            aws_session_token=get_first_property_value(properties, GLUE_SESSION_TOKEN, AWS_SESSION_TOKEN),
         )
         self.glue: GlueClient = session.client("glue", endpoint_url=properties.get(GLUE_CATALOG_ENDPOINT))
 
@@ -670,7 +659,7 @@ def drop_namespace(self, namespace: Union[str, Identifier]) -> None:
         self.glue.delete_database(Name=database_name)
 
     def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]:
-        """List tables under the given namespace in the catalog (including non-Iceberg tables).
+        """List Iceberg tables under the given namespace in the catalog.
 
         Args:
             namespace (str | Identifier): Namespace identifier to search.
@@ -698,7 +687,7 @@ def list_tables(self, namespace: Union[str, Identifier]) -> List[Identifier]:
 
         except self.glue.exceptions.EntityNotFoundException as e:
             raise NoSuchNamespaceError(f"Database does not exist: {database_name}") from e
-        return [(database_name, table["Name"]) for table in table_list]
+        return [(database_name, table["Name"]) for table in table_list if self.__is_iceberg_table(table)]
 
     def list_namespaces(self, namespace: Union[str, Identifier] = ()) -> List[Identifier]:
         """List namespaces from the given namespace. If not given, list top-level namespaces from the catalog.
@@ -781,3 +770,7 @@ def list_views(self, namespace: Union[str, Identifier]) -> List[Identifier]:
 
     def drop_view(self, identifier: Union[str, Identifier]) -> None:
         raise NotImplementedError
+
+    @staticmethod
+    def __is_iceberg_table(table: TableTypeDef) -> bool:
+        return table.get("Parameters", {}).get("table_type", "").lower() == ICEBERG