Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 23 additions & 21 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -32,18 +32,18 @@ classifiers = [
"Programming Language :: Python :: 3.14",
]
dependencies = [
"mmh3>=4.0.0,<6.0.0",
"requests>=2.20.0,<3.0.0",
"click>=7.1.1,<9.0.0",
"rich>=10.11.0,<16.0.0",
"strictyaml>=1.7.0,<2.0.0", # CVE-2020-14343 was fixed in 5.4.
"mmh3>=4.0.0",
"requests>=2.20.0",
"click>=7.1.1",
"rich>=10.11.0",
"strictyaml>=1.7.0", # CVE-2020-14343 was fixed in 5.4.
"pydantic>=2.0,<3.0,!=2.4.0,!=2.4.1,!=2.12.0,!=2.12.1", # 2.4.0, 2.4.1, 2.12.0, 2.12.1 has a critical bug
"fsspec>=2023.1.0",
"pyparsing>=3.1.0,<4.0.0",
"tenacity>=8.2.3,<10.0.0",
"pyroaring>=1.0.0,<2.0.0",
"cachetools>=5.5,<8.0",
"zstandard>=0.13.0,<1.0.0"
"pyparsing>=3.1.0",
"tenacity>=8.2.3",
"pyroaring>=1.0.0",
"cachetools>=5.5",
"zstandard>=0.13.0"
]

[project.urls]
Expand All @@ -62,23 +62,23 @@ pandas = [
"pyarrow>=17.0.0",
]
duckdb = [
"duckdb>=0.5.0,<2.0.0",
"duckdb>=0.5.0",
"pyarrow>=17.0.0",
]
ray = [
"ray>=2.10.0,<3.0.0",
"ray>=2.10.0",
"pyarrow>=17.0.0",
"pandas>=1.0.0",
]
bodo = ["bodo>=2025.7.4"]
daft = ["daft>=0.7.10"]
polars = ["polars>=1.21.0,<2"]
snappy = ["python-snappy>=0.6.0,<1.0.0"]
hive = ["thrift>=0.13.0,<1.0.0"]
polars = ["polars>=1.21.0"]
snappy = ["python-snappy>=0.6.0"]
hive = ["thrift>=0.13.0"]
hive-kerberos = [
"thrift>=0.13.0,<1.0.0",
"thrift>=0.13.0",
"thrift-sasl>=0.4.3",
"kerberos>=1.3.1,<2",
"kerberos>=1.3.1",
]
s3fs = ["s3fs>=2023.1.0"]
glue = ["boto3>=1.24.59"]
Expand All @@ -103,18 +103,18 @@ geoarrow = ["geoarrow-pyarrow>=0.2.0"]
dev = [
"pytest==9.1.1",
"pytest-checkdocs==2.14.0",
"prek>=0.2.1,<0.5",
"prek>=0.2.1",
"pytest-lazy-fixtures==1.4.0",
"fastavro==1.12.2",
"coverage[toml]>=7.4.2,<8",
"coverage[toml]>=7.4.2",
"requests-mock==1.12.1",
"moto[server]>=5.0.2,<6",
"moto[server]>=5.0.2",
"typing-extensions==4.15.0",
"pytest-mock==3.15.1",
"pyspark[connect]==4.0.1",
"protobuf==6.33.5", # match Spark Connect's gencode
"cython>=3.0.0",
"deptry>=0.14,<0.26",
"deptry>=0.14",
"docutils!=0.21.post1",
"mypy-boto3-glue>=1.28.18",
"mypy-boto3-dynamodb>=1.28.18",
Expand Down Expand Up @@ -178,6 +178,8 @@ filterwarnings = [
"ignore:subprocess.*is still running:ResourceWarning",
# Ignore google-crc32c C extension missing warning (common on Python 3.14+)
"ignore:As the c extension couldn't be imported:RuntimeWarning:google_crc32c",
# Ignore Spark 4.0.1 pandas conversion warning under pandas 3.0
"ignore:The copy keyword is deprecated and will be removed in a future version.*",
]

[tool.mypy]
Expand Down
4 changes: 2 additions & 2 deletions tests/catalog/test_glue.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def _db_as_dict_with_federated(self): # type: ignore
# warehouse location.
_original_table_init = FakeTable.__init__

def _table_init_with_location(self, database_name, table_name, table_input, catalog_id): # type: ignore
def _table_init_with_location(self, database_name, table_name, table_input, *args, **kwargs): # type: ignore

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not against these changes, but I think they are unrelated?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is because updated moto included a new open_table_format_input param
https://github.com/getmoto/moto/blame/09a8275d8330e359b0b35f5325e7740ed26552b7/moto/glue/models.py#L1715

passing args/kwargs through

if table_input.get("Parameters", {}).get("format") == "ICEBERG" and "StorageDescriptor" not in table_input:
table_input = {
**table_input,
Expand All @@ -84,7 +84,7 @@ def _table_init_with_location(self, database_name, table_name, table_input, cata
"SerdeInfo": {},
},
}
_original_table_init(self, database_name, table_name, table_input, catalog_id)
_original_table_init(self, database_name, table_name, table_input, *args, **kwargs)

monkeypatch.setattr(FakeTable, "__init__", _table_init_with_location)

Expand Down
2 changes: 1 addition & 1 deletion tests/integration/test_writes/test_writes.py
Original file line number Diff line number Diff line change
Expand Up @@ -768,7 +768,7 @@ def test_spark_writes_orc_pyiceberg_reads(spark: SparkSession, session_catalog:

# Verify PyIceberg data types are correct
assert pyiceberg_df["id"].dtype == "int64"
assert pyiceberg_df["name"].dtype == "object" # string
assert pd.api.types.is_string_dtype(pyiceberg_df["name"].dtype)
assert pyiceberg_df["age"].dtype == "int64"
assert pyiceberg_df["is_active"].dtype == "bool"

Expand Down
Loading