Python 3.11 support (#435)

* WIP: Python 3.11 support

* put in temp warning in test_frame.py

* fix pyright up to 1.1.279

* fix bounds test in test_frame.py:test_generic

* change bound on test to 3.10.99

* change strings to comments related to unavailability

* test with pandas 1.5.2, pyright 1.1.281

* remove pyarrow checks for python 3.11

* update github action to get rid of github set-output warning

Author: Dr-Irv

.github/setup/action.yaml

@@ -21,7 +21,7 @@ runs:
 
     - name: Determine poetry version
       shell: bash
-      run: echo "::set-output name=VERSION::$(poetry --version)"
+      run: echo "{VERSION}=$(poetry --version)"
       id: poetry_version
 
     - name: Cache poetry.lock

.github/workflows/test.yml

@@ -14,7 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, windows-latest, macos-latest]
-        python-version: ['3.8', '3.9', '3.10.6']
+        python-version: ['3.8', '3.9', '3.10.6', '3.11']
 
     steps:
     - uses: actions/checkout@v3

pyproject.toml

@@ -19,6 +19,7 @@ classifiers = [
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
     "Topic :: Scientific/Engineering"
 ]
 packages = [
@@ -30,26 +31,26 @@ packages = [
 "Documentation" = "https://pandas.pydata.org/pandas-docs/stable"
 
 [tool.poetry.dependencies]
-python = ">=3.8,<3.11"
+python = ">=3.8,<3.12"
 types-pytz = ">= 2022.1.1"
 
 [tool.poetry.dev-dependencies]
 mypy = "0.990"
-pyarrow = ">=9.0.0"
+pyarrow = ">=10.0.1"
 pytest = ">=7.1.2"
-pyright = ">=1.1.278"
+pyright = ">=1.1.281"
 poethepoet = "0.16.0"
 loguru = ">=0.6.0"
-pandas = "1.5.1"
+pandas = "1.5.2"
 typing-extensions = ">=4.2.0"
 matplotlib = ">=3.5.1"
 pre-commit = ">=2.19.0"
 black = ">=22.8.0"
 isort = ">=5.10.1"
 openpyxl = ">=3.0.10"
-tables = ">=3.7.0"
-lxml = ">=4.7.1,<4.9.0"
-pyreadstat = ">=1.1.9"
+tables = { version = ">=3.7.0", python = "<3.11" }
+lxml = { version = ">=4.7.1,<4.9.0", python = "<3.11" }
+pyreadstat = ">=1.2.0"
 xlrd = ">=2.0.1"
 pyxlsb = ">=1.0.9"
 odfpy = ">=1.4.1"

tests/__init__.py

@@ -6,6 +6,7 @@
 )
 import os
 import platform
+import sys
 from typing import (
     TYPE_CHECKING,
     Final,
@@ -21,6 +22,16 @@
 WINDOWS = os.name == "nt" or "cygwin" in platform.system().lower()
 PD_LTE_15 = Version(pd.__version__) < Version("1.5.999")
 
+lxml_skip = pytest.mark.skipif(
+    sys.version_info >= (3, 11), reason="lxml is not available for 3.11 yet"
+)
+# This is only needed temporarily due to no wheels being available for lxml on 3.11
+
+pytables_skip = pytest.mark.skipif(
+    sys.version_info >= (3, 11), reason="pytables is not available for 3.11 yet"
+)
+# This is only needed temporarily due to no wheels being available for pytables on 3.11
+
 
 def check(actual: T, klass: type, dtype: type | None = None, attr: str = "left") -> T:
     if not isinstance(actual, klass):
@@ -44,6 +55,7 @@ def pytest_warns_bounded(
     match: str,
     lower: str | None = None,
     upper: str | None = None,
+    version_str: str | None = None,
 ) -> AbstractContextManager:
     """
     Version conditional pytest.warns context manager
@@ -62,11 +74,14 @@ def pytest_warns_bounded(
         The lower bound of the version to check for the warning.
     upper : str, optional
         The upper bound of the version to check for the warning.
+    version_str: str, optional
+        The version string to use.  If None, then uses the pandas version.
+        Can be used to check a python version as well
 
     Notes
     -----
     The lower and upper bounds are exclusive so that a pytest.warns context
-    manager is returned if lower < pd.__version__ < upper.
+    manager is returned if lower < version_str < upper.
 
     Examples
     --------
@@ -85,10 +100,21 @@ def pytest_warns_bounded(
     ):
         # Versions between 1.3.x and 1.5.x will raise an error
         pass
+
+    with pytest_warns_bounded(
+        UserWarning, match="foo", lower="3.10",
+        version_str = platform.python_version()
+    ):
+        # Python version 3.11 and above will raise an error
+        # if the warning is not issued
+        pass
     """
     lb = Version("0.0.0") if lower is None else Version(lower)
     ub = Version("9999.0.0") if upper is None else Version(upper)
-    current = Version(pd.__version__)
+    if version_str is None:
+        current = Version(pd.__version__)
+    else:
+        current = Version(version_str)
     if lb < current < ub:
         return pytest.warns(warning, match=match)
     else:

tests/test_frame.py

@@ -6,6 +6,7 @@
 import io
 import itertools
 from pathlib import Path
+import platform
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -1757,7 +1758,15 @@ class MyDataFrame(pd.DataFrame, Generic[T]):
     def func() -> MyDataFrame[int]:
         return MyDataFrame[int]({"foo": [1, 2, 3]})
 
-    func()
+    # This should be fixed in pandas 1.5.2, if
+    # https://github.com/pandas-dev/pandas/pull/49736 is included
+    with pytest_warns_bounded(
+        UserWarning,
+        "Pandas doesn't allow columns to be created",
+        lower="3.10.99",
+        version_str=platform.python_version(),
+    ):
+        func()
 
 
 def test_to_xarray():

tests/test_io.py

@@ -65,6 +65,11 @@
 from pandas.io.sas.sas_xport import XportReader
 from pandas.io.stata import StataReader
 
+from . import (
+    lxml_skip,
+    pytables_skip,
+)
+
 DF = DataFrame({"a": [1, 2, 3], "b": [0.0, 0.0, 0.0]})
 CWD = os.path.split(os.path.abspath(__file__))[0]
 
@@ -108,6 +113,7 @@ def test_orc_bytes():
     check(assert_type(DF.to_orc(index=False), bytes), bytes)
 
 
+@lxml_skip
 def test_xml():
     with ensure_clean() as path:
         check(assert_type(DF.to_xml(path), None), type(None))
@@ -116,6 +122,7 @@ def test_xml():
             check(assert_type(read_xml(f), DataFrame), DataFrame)
 
 
+@lxml_skip
 def test_xml_str():
     with ensure_clean() as path:
         check(assert_type(DF.to_xml(), str), str)
@@ -282,12 +289,14 @@ def test_sas_xport() -> None:
         pass
 
 
+@pytables_skip
 def test_hdf():
     with ensure_clean() as path:
         check(assert_type(DF.to_hdf(path, "df"), None), type(None))
         check(assert_type(read_hdf(path), Union[DataFrame, Series]), DataFrame)
 
 
+@pytables_skip
 def test_hdfstore():
     with ensure_clean() as path:
         store = HDFStore(path, model="w")
@@ -329,6 +338,7 @@ def test_hdfstore():
         store.close()
 
 
+@pytables_skip
 def test_read_hdf_iterator():
     with ensure_clean() as path:
         check(assert_type(DF.to_hdf(path, "df", format="table"), None), type(None))
@@ -343,6 +353,7 @@ def test_read_hdf_iterator():
         ti.close()
 
 
+@pytables_skip
 def test_hdf_context_manager():
     with ensure_clean() as path:
         check(assert_type(DF.to_hdf(path, "df", format="table"), None), type(None))
@@ -351,6 +362,7 @@ def test_hdf_context_manager():
             check(assert_type(store.get("df"), Union[DataFrame, Series]), DataFrame)
 
 
+@pytables_skip
 def test_hdf_series():
     s = DF["a"]
     with ensure_clean() as path:
@@ -795,6 +807,7 @@ def test_read_sql_query_generator():
         con.close()
 
 
+@lxml_skip
 def test_read_html():
     check(assert_type(DF.to_html(), str), str)
     with ensure_clean() as path: