From c7ecf1dca9c0cfe16cfcbb5052c07a6955ef7330 Mon Sep 17 00:00:00 2001 From: Dave Tapley Date: Fri, 23 Oct 2026 10:34:34 -0700 Subject: [PATCH] Replace pytz with zoneinfo https://github.com/duckdb/duckdb-python/discussions/406 --- benchmarks/requirements-bench.txt | 1 - benchmarks/test_fetch_perf.py | 2 +- pyproject.toml | 3 -- scripts/cache_data.json | 28 +++++++++---------- scripts/imports.py | 4 +-- .../{pytz_module.hpp => zoneinfo_module.hpp} | 12 ++++---- .../import_cache/python_import_cache.hpp | 2 +- .../python_import_cache_modules.hpp | 4 +-- src/native/python_objects.cpp | 6 ++-- tests/fast/api/test_native_tz.py | 8 +++--- tests/fast/arrow/test_arrow_offsets.py | 10 +++---- tests/fast/arrow/test_timestamp_timezone.py | 4 +-- tests/fast/pandas/test_append_df.py | 7 +++-- tests/fast/test_all_types.py | 10 +++---- tests/fast/test_optional_dependencies.py | 14 +++------- 15 files changed, 53 insertions(+), 62 deletions(-) rename src/include/duckdb_python/import_cache/modules/{pytz_module.hpp => zoneinfo_module.hpp} (67%) diff --git a/benchmarks/requirements-bench.txt b/benchmarks/requirements-bench.txt index 8a9f49e68..50a6edee9 100644 --- a/benchmarks/requirements-bench.txt +++ b/benchmarks/requirements-bench.txt @@ -18,6 +18,5 @@ pygments==2.20.0 pytest==9.1.1 pytest-codspeed==5.0.3 python-dateutil==2.9.0.post0 -pytz==2026.2 rich==15.0.0 six==1.17.0 diff --git a/benchmarks/test_fetch_perf.py b/benchmarks/test_fetch_perf.py index 1b07dfb74..46848d790 100644 --- a/benchmarks/test_fetch_perf.py +++ b/benchmarks/test_fetch_perf.py @@ -67,7 +67,7 @@ def run() -> None: # small-N *_gate variants: at range(2048) the measured region is ~60% SQL compile + engine, ~40% fetch, so these -# catch a fixed-cost regression (not a pure per-row one). Plus expensive scalar types (timestamptz pytz-per-row, +# catch a fixed-cost regression (not a pure per-row one). Plus expensive scalar types (timestamptz per-row, # blob, null-heavy), a heterogeneous per-cell-dispatch row, and the batched fetchmany loop. diff --git a/pyproject.toml b/pyproject.toml index b3e7cb776..c408529aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -291,7 +291,6 @@ test = [ # dependencies used for running tests "py4j", "pyotp", "pyspark", - "pytz", "requests", "urllib3", "fsspec>=2022.11.0; sys_platform != 'win32' or platform_machine != 'ARM64'", @@ -316,7 +315,6 @@ bench = [ # Pinned deps for the benchmark suite (see benchmarks/README.md). Mini "pytest", "pytest_codspeed", "polars>=1.33.0", - "pytz", "numpy<2; ( sys_platform != 'win32' or platform_machine != 'ARM64' ) and python_version < '3.12'", "numpy>=2; ( sys_platform != 'win32' or platform_machine != 'ARM64' ) and python_version >= '3.12'", "numpy>=2.3; sys_platform == 'win32' and platform_machine == 'ARM64'", @@ -331,7 +329,6 @@ scripts = [ # dependencies used for running scripts "pandas", "polars>=1.33.0", "pyarrow; sys_platform != 'win32' or platform_machine != 'ARM64'", - "pytz" ] pypi = [ # dependencies used by the pypi cleanup script "pyotp>=2.9.0", diff --git a/scripts/cache_data.json b/scripts/cache_data.json index a822fa81d..124fd0d95 100644 --- a/scripts/cache_data.json +++ b/scripts/cache_data.json @@ -582,20 +582,6 @@ "name": "Value", "children": [] }, - "pytz": { - "type": "module", - "full_path": "pytz", - "name": "pytz", - "children": [ - "pytz.timezone" - ] - }, - "pytz.timezone": { - "type": "attribute", - "full_path": "pytz.timezone", - "name": "timezone", - "children": [] - }, "types": { "type": "module", "full_path": "types", @@ -858,5 +844,19 @@ "full_path": "pandas.arrays.StringArray", "name": "StringArray", "children": [] + }, + "zoneinfo": { + "type": "module", + "full_path": "zoneinfo", + "name": "zoneinfo", + "children": [ + "zoneinfo.ZoneInfo" + ] + }, + "zoneinfo.ZoneInfo": { + "type": "attribute", + "full_path": "zoneinfo.ZoneInfo", + "name": "ZoneInfo", + "children": [] } } \ No newline at end of file diff --git a/scripts/imports.py b/scripts/imports.py index 477e8a8f9..4f0cc0070 100644 --- a/scripts/imports.py +++ b/scripts/imports.py @@ -122,9 +122,9 @@ duckdb.filesystem.ModifiedMemoryFileSystem duckdb.Value -import pytz +import zoneinfo -pytz.timezone +zoneinfo.ZoneInfo import types diff --git a/src/include/duckdb_python/import_cache/modules/pytz_module.hpp b/src/include/duckdb_python/import_cache/modules/zoneinfo_module.hpp similarity index 67% rename from src/include/duckdb_python/import_cache/modules/pytz_module.hpp rename to src/include/duckdb_python/import_cache/modules/zoneinfo_module.hpp index ccdde0c89..5671433dd 100644 --- a/src/include/duckdb_python/import_cache/modules/pytz_module.hpp +++ b/src/include/duckdb_python/import_cache/modules/zoneinfo_module.hpp @@ -2,7 +2,7 @@ //===----------------------------------------------------------------------===// // DuckDB // -// duckdb_python/import_cache/modules/pytz_module.hpp +// duckdb_python/import_cache/modules/zoneinfo_module.hpp // // //===----------------------------------------------------------------------===// @@ -20,18 +20,18 @@ namespace duckdb { -struct PytzCacheItem : public PythonImportCacheItem { +struct ZoneinfoCacheItem : public PythonImportCacheItem { public: - static constexpr const char *Name = "pytz"; + static constexpr const char *Name = "zoneinfo"; public: - PytzCacheItem() : PythonImportCacheItem("pytz"), timezone("timezone", this) { + ZoneinfoCacheItem() : PythonImportCacheItem("zoneinfo"), ZoneInfo("ZoneInfo", this) { } - ~PytzCacheItem() override { + ~ZoneinfoCacheItem() override { } - PythonImportCacheItem timezone; + PythonImportCacheItem ZoneInfo; }; } // namespace duckdb diff --git a/src/include/duckdb_python/import_cache/python_import_cache.hpp b/src/include/duckdb_python/import_cache/python_import_cache.hpp index 6438332f6..bbe67d00b 100644 --- a/src/include/duckdb_python/import_cache/python_import_cache.hpp +++ b/src/include/duckdb_python/import_cache/python_import_cache.hpp @@ -33,11 +33,11 @@ struct PythonImportCache { PathlibCacheItem pathlib; PolarsCacheItem polars; DuckdbCacheItem duckdb; - PytzCacheItem pytz; TypesCacheItem types; TypingCacheItem typing; UuidCacheItem uuid; CollectionsCacheItem collections; + ZoneinfoCacheItem zoneinfo; public: nb::handle AddCache(nb::object item); diff --git a/src/include/duckdb_python/import_cache/python_import_cache_modules.hpp b/src/include/duckdb_python/import_cache/python_import_cache_modules.hpp index e82cd2dcc..f500b8995 100644 --- a/src/include/duckdb_python/import_cache/python_import_cache_modules.hpp +++ b/src/include/duckdb_python/import_cache/python_import_cache_modules.hpp @@ -8,8 +8,8 @@ #include "duckdb_python/import_cache/modules/pathlib_module.hpp" #include "duckdb_python/import_cache/modules/polars_module.hpp" #include "duckdb_python/import_cache/modules/duckdb_module.hpp" -#include "duckdb_python/import_cache/modules/pytz_module.hpp" #include "duckdb_python/import_cache/modules/types_module.hpp" #include "duckdb_python/import_cache/modules/typing_module.hpp" #include "duckdb_python/import_cache/modules/uuid_module.hpp" -#include "duckdb_python/import_cache/modules/collections_module.hpp" \ No newline at end of file +#include "duckdb_python/import_cache/modules/collections_module.hpp" +#include "duckdb_python/import_cache/modules/zoneinfo_module.hpp" \ No newline at end of file diff --git a/src/native/python_objects.cpp b/src/native/python_objects.cpp index 2b476ddd6..9653407d2 100644 --- a/src/native/python_objects.cpp +++ b/src/native/python_objects.cpp @@ -662,9 +662,9 @@ nb::object PythonObject::FromValue(const Value &val, const LogicalType &type, } if (type.id() == LogicalTypeId::TIMESTAMP_TZ || type.id() == LogicalTypeId::TIMESTAMP_TZ_NS) { // We have to add the timezone info - auto tz_utc = import_cache.pytz.timezone()("UTC"); - auto timestamp_utc = tz_utc.attr("localize")(py_timestamp); - auto tz_info = import_cache.pytz.timezone()(client_properties.time_zone); + auto tz_utc = import_cache.zoneinfo.ZoneInfo()("UTC"); + auto timestamp_utc = py_timestamp.attr("replace")(nb::arg("tzinfo") = tz_utc); + auto tz_info = import_cache.zoneinfo.ZoneInfo()(client_properties.time_zone); return timestamp_utc.attr("astimezone")(tz_info); } return py_timestamp; diff --git a/tests/fast/api/test_native_tz.py b/tests/fast/api/test_native_tz.py index 0545182ee..d869bffa1 100644 --- a/tests/fast/api/test_native_tz.py +++ b/tests/fast/api/test_native_tz.py @@ -30,23 +30,23 @@ def test_native_python_timestamp_timezone(self, duckdb_cursor): res = duckdb_cursor.execute(f"select TimeRecStart as tz from '{filename}'").fetchone() assert res[0].hour == 14 assert res[0].minute == 52 - assert res[0].tzinfo.zone == "America/Los_Angeles" + assert res[0].tzinfo.key == "America/Los_Angeles" res = duckdb_cursor.execute(f"select TimeRecStart as tz from '{filename}'").fetchall()[0] assert res[0].hour == 14 assert res[0].minute == 52 - assert res[0].tzinfo.zone == "America/Los_Angeles" + assert res[0].tzinfo.key == "America/Los_Angeles" res = duckdb_cursor.execute(f"select TimeRecStart as tz from '{filename}'").fetchmany(1)[0] assert res[0].hour == 14 assert res[0].minute == 52 - assert res[0].tzinfo.zone == "America/Los_Angeles" + assert res[0].tzinfo.key == "America/Los_Angeles" duckdb_cursor.execute("SET timezone='UTC';") res = duckdb_cursor.execute(f"select TimeRecStart as tz from '{filename}'").fetchone() assert res[0].hour == 21 assert res[0].minute == 52 - assert res[0].tzinfo.zone == "UTC" + assert res[0].tzinfo.key == "UTC" def test_native_python_time_timezone(self, duckdb_cursor): res = duckdb_cursor.execute(f"select TimeRecStart::TIMETZ as tz from '{filename}'").fetchone() diff --git a/tests/fast/arrow/test_arrow_offsets.py b/tests/fast/arrow/test_arrow_offsets.py index 304b87958..d352066b6 100644 --- a/tests/fast/arrow/test_arrow_offsets.py +++ b/tests/fast/arrow/test_arrow_offsets.py @@ -1,9 +1,9 @@ # ruff: noqa: F841 import datetime import decimal +from zoneinfo import ZoneInfo import pytest -import pytz pa = pytest.importorskip("pyarrow") @@ -324,10 +324,10 @@ def test_struct_of_duration(self, duckdb_cursor, constructor, unit, expected, co @pytest.mark.parametrize( ("constructor", "unit", "expected"), [ - (pa_timestamp(), "ms", datetime.datetime(1970, 1, 1, 0, 2, 11, 72000, tzinfo=pytz.utc)), - (pa_timestamp(), "s", datetime.datetime(1970, 1, 2, 12, 24, 32, 0, tzinfo=pytz.utc)), - (pa_timestamp(), "ns", datetime.datetime(1970, 1, 1, 0, 0, 0, 131, tzinfo=pytz.utc)), - (pa_timestamp(), "us", datetime.datetime(1970, 1, 1, 0, 0, 0, 131072, tzinfo=pytz.utc)), + (pa_timestamp(), "ms", datetime.datetime(1970, 1, 1, 0, 2, 11, 72000, tzinfo=ZoneInfo("UTC"))), + (pa_timestamp(), "s", datetime.datetime(1970, 1, 2, 12, 24, 32, 0, tzinfo=ZoneInfo("UTC"))), + (pa_timestamp(), "ns", datetime.datetime(1970, 1, 1, 0, 0, 0, 131, tzinfo=ZoneInfo("UTC"))), + (pa_timestamp(), "us", datetime.datetime(1970, 1, 1, 0, 0, 0, 131072, tzinfo=ZoneInfo("UTC"))), ], ) def test_struct_of_timestamp_tz(self, duckdb_cursor, constructor, unit, expected, col1_null, col2_null): diff --git a/tests/fast/arrow/test_timestamp_timezone.py b/tests/fast/arrow/test_timestamp_timezone.py index c847c28d6..b81e964a4 100644 --- a/tests/fast/arrow/test_timestamp_timezone.py +++ b/tests/fast/arrow/test_timestamp_timezone.py @@ -1,7 +1,7 @@ import datetime +from zoneinfo import ZoneInfo import pytest -import pytz import duckdb @@ -21,7 +21,7 @@ def generate_table(current_time, precision, timezone): class TestArrowTimestampsTimezone: def test_timestamp_timezone(self, duckdb_cursor): precisions = ["us", "s", "ns", "ms"] - current_time = datetime.datetime(2017, 11, 28, 23, 55, 59, tzinfo=pytz.UTC) + current_time = datetime.datetime(2017, 11, 28, 23, 55, 59, tzinfo=ZoneInfo("UTC")) con = duckdb.connect() con.execute("SET TimeZone = 'UTC'") for precision in precisions: diff --git a/tests/fast/pandas/test_append_df.py b/tests/fast/pandas/test_append_df.py index 839fe4085..52dc1a8bd 100644 --- a/tests/fast/pandas/test_append_df.py +++ b/tests/fast/pandas/test_append_df.py @@ -3,9 +3,9 @@ import math import pathlib from decimal import Decimal +from zoneinfo import ZoneInfo import pytest -import pytz import duckdb import numpy as np @@ -416,7 +416,7 @@ def test_naive_datetime_into_timestamptz_reads_in_the_session_time_zone(self, du duckdb_cursor.execute("CREATE TABLE t (a TIMESTAMPTZ)") duckdb_cursor.append("t", pd.DataFrame({"a": _outside_the_sample(AWARE, naive)})) values = _column(duckdb_cursor) - assert (values[0], values[-1]) == (AWARE, pytz.timezone(zone).localize(NAIVE)) + assert (values[0], values[-1]) == (AWARE, NAIVE.replace(tzinfo=ZoneInfo(zone))) @pytest.mark.parametrize( "zone", @@ -439,7 +439,8 @@ def test_date_into_timestamptz_reads_as_midnight_in_the_session_time_zone(self, duckdb_cursor.execute("CREATE TABLE t (a TIMESTAMPTZ)") duckdb_cursor.append("t", pd.DataFrame({"a": _outside_the_sample(AWARE, datetime.date(2024, 5, 6))})) values = _column(duckdb_cursor) - assert (values[0], values[-1]) == (AWARE, pytz.timezone(zone).localize(datetime.datetime(2024, 5, 6))) + expected = datetime.datetime(2024, 5, 6, tzinfo=ZoneInfo(zone)) + assert (values[0], values[-1]) == (AWARE, expected) @pytest.mark.xfail(strict=True, reason=NOT_YET_DECLARED) def test_by_name_declares_each_column_from_its_name(self, duckdb_cursor): diff --git a/tests/fast/test_all_types.py b/tests/fast/test_all_types.py index f346fa903..fe7a1d747 100644 --- a/tests/fast/test_all_types.py +++ b/tests/fast/test_all_types.py @@ -4,9 +4,9 @@ from contextlib import suppress from decimal import Decimal from uuid import UUID +from zoneinfo import ZoneInfo import pytest -import pytz import duckdb import numpy as np @@ -133,9 +133,9 @@ def test_fetchall(self, cur_type): "time_tz": """CASE WHEN time_tz = '24:00:00-1559'::TIMETZ THEN '23:59:59.999999-1559'::TIMETZ ELSE time_tz END AS "time_tz" """, # noqa: E501 } min_datetime = datetime.datetime.min - min_datetime_with_utc = min_datetime.replace(tzinfo=pytz.UTC) + min_datetime_with_utc = min_datetime.replace(tzinfo=ZoneInfo("UTC")) max_datetime = datetime.datetime.max - max_datetime_with_utc = max_datetime.replace(tzinfo=pytz.UTC) + max_datetime_with_utc = max_datetime.replace(tzinfo=ZoneInfo("UTC")) correct_answer_map = { "bool": [(False,), (True,), (None,)], "tinyint": [(-128,), (127,), (None,)], @@ -196,7 +196,7 @@ def test_fetchall(self, cur_type): ( [], [ - datetime.datetime(1970, 1, 1, tzinfo=pytz.UTC), + datetime.datetime(1970, 1, 1, tzinfo=ZoneInfo("UTC")), None, min_datetime_with_utc, max_datetime_with_utc, @@ -237,7 +237,7 @@ def test_fetchall(self, cur_type): "timestamp_s": [(datetime.datetime(1990, 1, 1, 0, 0),)], "timestamp_ns": [(datetime.datetime(1990, 1, 1, 0, 0),)], "timestamp_ms": [(datetime.datetime(1990, 1, 1, 0, 0),)], - "timestamp_tz": [(datetime.datetime(1990, 1, 1, 0, 0, tzinfo=pytz.UTC),)], + "timestamp_tz": [(datetime.datetime(1990, 1, 1, 0, 0, tzinfo=ZoneInfo("UTC")),)], "union": [("Frank",), (5,), (None,)], "fixed_int_array": [((None, 2, 3),), ((4, 5, 6),), (None,)], "fixed_varchar_array": [(("a", None, "c"),), (("d", "e", "f"),), (None,)], diff --git a/tests/fast/test_optional_dependencies.py b/tests/fast/test_optional_dependencies.py index 2cacc71f3..cf1bec6de 100644 --- a/tests/fast/test_optional_dependencies.py +++ b/tests/fast/test_optional_dependencies.py @@ -11,7 +11,6 @@ "pyarrow", "polars", "fsspec", - "pytz", "typing_extensions", "pyspark", "IPython", @@ -162,7 +161,7 @@ def test_spark_timestamps(tmp_path): def test_spark_timestamps_to_pandas(tmp_path): - # pandas before 3.0 cannot be imported without pytz. + # pandas before 3.0 cannot be imported when pytz is blocked. pytest.importorskip("pandas", minversion="3.0.0") run_blocked( tmp_path, @@ -201,8 +200,8 @@ def test_spark_udf_needs_numpy(tmp_path, blocked): ) -def test_spark_udf_with_a_timestamp_needs_pytz(tmp_path): - """The binding's Python UDF path needs pytz to pass a timestamp to a function.""" +def test_spark_udf_with_a_timestamp_works_without_pytz(tmp_path): + """The binding's Python UDF path uses zoneinfo, not pytz, to pass a timestamp to a function.""" if importlib.util.find_spec("numpy") is None: pytest.skip("a UDF needs numpy") run_blocked( @@ -218,12 +217,7 @@ def test_spark_udf_with_a_timestamp_needs_pytz(tmp_path): assert spark.createDataFrame([(1,)], "n int").select(plus_one("n")).collect()[0][0] == 2 year = spark.udf.register("year_of", lambda t: t.year, "int") df = spark.createDataFrame([(datetime.datetime(2024, 1, 2),)], "t timestamp") - try: - df.select(year("t")).collect() - except duckdb.InvalidInputException as error: - assert "'pytz' failed to import" in str(error), repr(error) - else: - raise AssertionError("a UDF of a timestamp without pytz did not raise") + assert df.select(year("t")).collect()[0][0] == 2024 """, ("pytz",), )