Adding support for Pandas 1.1.0.

Author: igorborgest

awswrangler/athena/_utils.py

@@ -127,7 +127,7 @@ def _fetch_txt_result(query_metadata: _QueryMetadata, keep_files: bool, boto3_se
         na_values=[],
         use_threads=False,
         boto3_session=boto3_session,
-        names=query_metadata.dtype.keys(),
+        names=list(query_metadata.dtype.keys()),
         sep="\t",
     )
     if keep_files is False:

awswrangler/s3/_write_parquet.py

@@ -149,6 +149,8 @@ def to_parquet(  # pylint: disable=too-many-arguments,too-many-locals
         https://github.com/awslabs/aws-data-wrangler/blob/master/tutorials/022%20-%20Writing%20Partitions%20Concurrently.ipynb
     mode: str, optional
         ``append`` (Default), ``overwrite``, ``overwrite_partitions``. Only takes effect if dataset=True.
+        For details check the related tutorial:
+        https://aws-data-wrangler.readthedocs.io/en/latest/stubs/awswrangler.s3.to_parquet.html#awswrangler.s3.to_parquet
     catalog_versioning : bool
         If True and `mode="overwrite"`, creates an archived version of the table catalog before updating it.
     database : str, optional

awswrangler/s3/_write_text.py

@@ -147,6 +147,8 @@ def to_csv(  # pylint: disable=too-many-arguments,too-many-locals
         https://github.com/awslabs/aws-data-wrangler/blob/master/tutorials/022%20-%20Writing%20Partitions%20Concurrently.ipynb
     mode : str, optional
         ``append`` (Default), ``overwrite``, ``overwrite_partitions``. Only takes effect if dataset=True.
+        For details check the related tutorial:
+        https://aws-data-wrangler.readthedocs.io/en/latest/stubs/awswrangler.s3.to_parquet.html#awswrangler.s3.to_parquet
     catalog_versioning : bool
         If True and `mode="overwrite"`, creates an archived version of the table catalog before updating it.
     database : str, optional

requirements-dev.txt

@@ -7,15 +7,15 @@ flake8==3.8.3
 mypy==0.782
 pydocstyle==5.0.2
 doc8==0.8.1
-tox==3.18.0
+tox==3.18.1
 pytest==5.4.3
 pytest-cov==2.10.0
-pytest-xdist==1.33.0
+pytest-xdist==1.34.0
 scikit-learn==0.23.1
 cfn-lint==0.34.0
 cfn-flip==1.2.3
 twine==3.2.0
 sphinx==3.1.2
 sphinx_bootstrap_theme==0.7.1
 moto==1.3.14
-jupyterlab~=2.2.0
+jupyterlab~=2.2.2

requirements.sh

@@ -2,5 +2,5 @@
 set -ex
 
 pip install --upgrade pip
-pip install --upgrade --requirement requirements-dev.txt
-pip install --upgrade --editable .
+pip install --upgrade --upgrade-strategy eager --requirement requirements-dev.txt
+pip install --upgrade --upgrade-strategy eager --editable .

requirements.txt

@@ -3,8 +3,8 @@ botocore>=1.15.49,<2.0.0
 numpy>=1.18.0,<1.20.0
 pymysql>=0.9.0,<0.11.0
 sqlalchemy-redshift>=0.7.0,<0.9.0
+pandas>=1.0.0,<1.2.0
 SQLAlchemy~=1.3.10
-pandas~=1.0.0
 pyarrow~=1.0.0
-s3fs~=0.4.2
 psycopg2-binary~=2.8.0
+s3fs==0.4.2

tests/test_athena.py

@@ -765,3 +765,8 @@ def test_parse_describe_table():
     parsed_df = wr.athena._utils._parse_describe_table(df)
     assert parsed_df["Partition"].to_list() == [False, False, False, True, True]
     assert parsed_df["Column Name"].to_list() == ["iint8", "iint16", "iint32", "par0", "par1"]
+
+
+def test_describe_table(path, glue_database, glue_table):
+    wr.catalog.create_parquet_table(database=glue_database, table=glue_table, path=path, columns_types={"c0": "int"})
+    assert wr.athena.describe_table(database=glue_database, table=glue_table).shape == (1, 4)