From 76bfcd8d854fd150a145ecff53f0a43a12151d46 Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 28 Dec 2022 17:30:44 -0800 Subject: [PATCH 1/4] DEP: Bump pytest, xdist, hypothesis --- .github/workflows/32-bit-linux.yml | 2 +- .github/workflows/python-dev.yml | 2 +- .github/workflows/wheels.yml | 2 +- ci/deps/actions-310-numpydev.yaml | 6 +++--- ci/deps/actions-310.yaml | 4 ++-- ci/deps/actions-38-downstream_compat.yaml | 4 ++-- ci/deps/actions-38-minimum_versions.yaml | 6 +++--- ci/deps/actions-38.yaml | 4 ++-- ci/deps/actions-39.yaml | 4 ++-- ci/deps/actions-pypy-38.yaml | 6 +++--- ci/deps/circle-38-arm64.yaml | 4 ++-- ci/test_wheels_windows.bat | 4 ++-- doc/source/getting_started/install.rst | 4 ++-- doc/source/whatsnew/v2.0.0.rst | 20 +++++++++++++------- environment.yml | 4 ++-- pandas/compat/_optional.py | 2 +- pyproject.toml | 10 +++++----- requirements-dev.txt | 4 ++-- 18 files changed, 49 insertions(+), 43 deletions(-) diff --git a/.github/workflows/32-bit-linux.yml b/.github/workflows/32-bit-linux.yml index 438d2c7b4174e..fc00dcf73cbdd 100644 --- a/.github/workflows/32-bit-linux.yml +++ b/.github/workflows/32-bit-linux.yml @@ -39,7 +39,7 @@ jobs: . ~/virtualenvs/pandas-dev/bin/activate && \ python -m pip install --no-deps -U pip wheel 'setuptools<60.0.0' && \ python -m pip install versioneer[toml] && \ - python -m pip install cython numpy python-dateutil pytz pytest pytest-xdist pytest-asyncio>=0.17 hypothesis && \ + python -m pip install cython numpy python-dateutil pytz pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 hypothesis>=6.34.2 && \ python setup.py build_ext -q -j1 && \ python -m pip install --no-build-isolation --no-use-pep517 -e . && \ python -m pip list && \ diff --git a/.github/workflows/python-dev.yml b/.github/workflows/python-dev.yml index 220c1e464742e..a0030517d1e99 100644 --- a/.github/workflows/python-dev.yml +++ b/.github/workflows/python-dev.yml @@ -76,7 +76,7 @@ jobs: python -m pip install -i https://pypi.anaconda.org/scipy-wheels-nightly/simple numpy python -m pip install git+https://github.com/nedbat/coveragepy.git python -m pip install versioneer[toml] - python -m pip install python-dateutil pytz cython hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-cov pytest-asyncio>=0.17 + python -m pip install python-dateutil pytz cython hypothesis>=6.34.2 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-cov pytest-asyncio>=0.17 python -m pip list # GH 47305: Parallel build can cause flaky ImportError from pandas/_libs/tslibs diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 49d29c91f86cd..f1dc15e1b3e9f 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -165,7 +165,7 @@ jobs: # (1. Generate sdist 2. Build wheels from sdist) # This tests the sdists, and saves some build time python -m pip install dist/*.gz - pip install hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-asyncio>=0.17 + pip install hypothesis>=6.34.2 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 cd .. # Not a good idea to test within the src tree python -c "import pandas; print(pandas.__version__); pandas.test(extra_args=['-m not clipboard and not single_cpu', '--skip-slow', '--skip-network', '--skip-db', '-n=2']); diff --git a/ci/deps/actions-310-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml index 863c231b18c4f..00db86b09d73e 100644 --- a/ci/deps/actions-310-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -8,10 +8,10 @@ dependencies: - versioneer[toml] # test dependencies - - pytest>=6.0 + - pytest>=7.0.0 - pytest-cov - - pytest-xdist>=1.31 - - hypothesis>=5.5.3 + - pytest-xdist>=2.2.0 + - hypothesis>=6.34.2 - pytest-asyncio>=0.17 # pandas dependencies diff --git a/ci/deps/actions-310.yaml b/ci/deps/actions-310.yaml index d787571d9d112..1b8dcf2eeed0d 100644 --- a/ci/deps/actions-310.yaml +++ b/ci/deps/actions-310.yaml @@ -9,9 +9,9 @@ dependencies: - cython>=0.29.32 # test dependencies - - pytest>=6.0 + - pytest>=7.0.0 - pytest-cov - - pytest-xdist>=1.31 + - pytest-xdist>=2.2.0 - psutil - pytest-asyncio>=0.17 - boto3 diff --git a/ci/deps/actions-38-downstream_compat.yaml b/ci/deps/actions-38-downstream_compat.yaml index 95ec98d72ebcc..7199604f7eb90 100644 --- a/ci/deps/actions-38-downstream_compat.yaml +++ b/ci/deps/actions-38-downstream_compat.yaml @@ -10,9 +10,9 @@ dependencies: - cython>=0.29.32 # test dependencies - - pytest>=6.0 + - pytest>=7.0.0 - pytest-cov - - pytest-xdist>=1.31 + - pytest-xdist>=2.2.0 - psutil - pytest-asyncio>=0.17 - boto3 diff --git a/ci/deps/actions-38-minimum_versions.yaml b/ci/deps/actions-38-minimum_versions.yaml index de7e793c46d19..59050431929e2 100644 --- a/ci/deps/actions-38-minimum_versions.yaml +++ b/ci/deps/actions-38-minimum_versions.yaml @@ -11,9 +11,9 @@ dependencies: - cython>=0.29.32 # test dependencies - - pytest>=6.0 + - pytest>=7.0.0 - pytest-cov - - pytest-xdist>=1.31 + - pytest-xdist>=2.2.0 - psutil - pytest-asyncio>=0.17 - boto3 @@ -31,7 +31,7 @@ dependencies: - fastparquet=0.6.3 - fsspec=2021.07.0 - html5lib=1.1 - - hypothesis=6.13.0 + - hypothesis==6.34.2 - gcsfs=2021.07.0 - jinja2=3.0.0 - lxml=4.6.3 diff --git a/ci/deps/actions-38.yaml b/ci/deps/actions-38.yaml index f7de8bbee7d8a..778c96906c5fe 100644 --- a/ci/deps/actions-38.yaml +++ b/ci/deps/actions-38.yaml @@ -9,9 +9,9 @@ dependencies: - cython>=0.29.32 # test dependencies - - pytest>=6.0 + - pytest>=7.0.0 - pytest-cov - - pytest-xdist>=1.31 + - pytest-xdist>=2.2.0 - psutil - pytest-asyncio>=0.17 - boto3 diff --git a/ci/deps/actions-39.yaml b/ci/deps/actions-39.yaml index 821ec9c5d4234..e11eede484e0a 100644 --- a/ci/deps/actions-39.yaml +++ b/ci/deps/actions-39.yaml @@ -9,9 +9,9 @@ dependencies: - cython>=0.29.32 # test dependencies - - pytest>=6.0 + - pytest>=7.0.0 - pytest-cov - - pytest-xdist>=1.31 + - pytest-xdist>=2.2.0 - psutil - pytest-asyncio>=0.17 - boto3 diff --git a/ci/deps/actions-pypy-38.yaml b/ci/deps/actions-pypy-38.yaml index 054129c4198a1..802b3f4108362 100644 --- a/ci/deps/actions-pypy-38.yaml +++ b/ci/deps/actions-pypy-38.yaml @@ -12,11 +12,11 @@ dependencies: - cython>=0.29.32 # test dependencies - - pytest>=6.0 + - pytest>=7.0.0 - pytest-cov - pytest-asyncio - - pytest-xdist>=1.31 - - hypothesis>=5.5.3 + - pytest-xdist>=2.2.0 + - hypothesis>=6.34.2 # required - numpy<1.24 diff --git a/ci/deps/circle-38-arm64.yaml b/ci/deps/circle-38-arm64.yaml index c94ce79ea2ff8..d3fde410c6a14 100644 --- a/ci/deps/circle-38-arm64.yaml +++ b/ci/deps/circle-38-arm64.yaml @@ -9,9 +9,9 @@ dependencies: - cython>=0.29.32 # test dependencies - - pytest>=6.0 + - pytest>=7.0.0 - pytest-cov - - pytest-xdist>=1.31 + - pytest-xdist>=2.2.0 - psutil - pytest-asyncio>=0.17 - boto3 diff --git a/ci/test_wheels_windows.bat b/ci/test_wheels_windows.bat index 9f4ebdabbf8d1..02f2eaf5d2ce7 100644 --- a/ci/test_wheels_windows.bat +++ b/ci/test_wheels_windows.bat @@ -4,6 +4,6 @@ pd.test(extra_args=['-m not clipboard and single_cpu', '--skip-slow', '--skip-ne python --version pip install pytz six numpy python-dateutil -pip install hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-asyncio>=0.17 +pip install hypothesis==6.52.1 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 pip install --find-links=pandas/dist --no-index pandas -python -c "%test_command%" +python -c "%test_command%" diff --git a/doc/source/getting_started/install.rst b/doc/source/getting_started/install.rst index 68065c77f7881..70a27a84a7592 100644 --- a/doc/source/getting_started/install.rst +++ b/doc/source/getting_started/install.rst @@ -208,8 +208,8 @@ pandas is equipped with an exhaustive set of unit tests, covering about 97% of the code base as of this writing. To run it on your machine to verify that everything is working (and that you have all of the dependencies, soft and hard, installed), make sure you have `pytest -`__ >= 6.0 and `Hypothesis -`__ >= 6.13.0, then run: +`__ >= 7.0 and `Hypothesis +`__ >= 6.34.2, then run: :: diff --git a/doc/source/whatsnew/v2.0.0.rst b/doc/source/whatsnew/v2.0.0.rst index a59ff15479771..e2dfb0ca2d63d 100644 --- a/doc/source/whatsnew/v2.0.0.rst +++ b/doc/source/whatsnew/v2.0.0.rst @@ -399,13 +399,19 @@ Increased minimum versions for dependencies Some minimum supported versions of dependencies were updated. If installed, we now require: -+-----------------+-----------------+----------+---------+ -| Package | Minimum Version | Required | Changed | -+=================+=================+==========+=========+ -| mypy (dev) | 0.991 | | X | -+-----------------+-----------------+----------+---------+ -| python-dateutil | 2.8.2 | X | X | -+-----------------+-----------------+----------+---------+ ++-------------------+-----------------+----------+---------+ +| Package | Minimum Version | Required | Changed | ++===================+=================+==========+=========+ +| mypy (dev) | 0.991 | | X | ++-------------------+-----------------+----------+---------+ +| pytest (dev) | 7.0.0 | | X | ++-------------------+-----------------+----------+---------+ +| pytest-xdist (dev)| 2.2.0 | | X | ++-------------------+-----------------+----------+---------+ +| hypothesis (dev) | 6.34.2 | | X | ++-------------------+-----------------+----------+---------+ +| python-dateutil | 2.8.2 | X | X | ++-------------------+-----------------+----------+---------+ For `optional libraries `_ the general recommendation is to use the latest version. The following table lists the lowest version per library that is currently being tested throughout the development of pandas. diff --git a/environment.yml b/environment.yml index b6b8f7d6af1ba..d3a4c111cf371 100644 --- a/environment.yml +++ b/environment.yml @@ -11,9 +11,9 @@ dependencies: - cython=0.29.32 # test dependencies - - pytest>=6.0 + - pytest>=7.0.0 - pytest-cov - - pytest-xdist>=1.31 + - pytest-xdist>=2.2.0 - psutil - pytest-asyncio>=0.17 - coverage diff --git a/pandas/compat/_optional.py b/pandas/compat/_optional.py index 9bd4b384fadb0..ce44c80c6fa52 100644 --- a/pandas/compat/_optional.py +++ b/pandas/compat/_optional.py @@ -19,7 +19,7 @@ "fastparquet": "0.6.3", "fsspec": "2021.07.0", "html5lib": "1.1", - "hypothesis": "6.13.0", + "hypothesis": "6.34.2", "gcsfs": "2021.07.0", "jinja2": "3.0.0", "lxml.etree": "4.6.3", diff --git a/pyproject.toml b/pyproject.toml index 385c1beb08121..11d28a366045c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,7 +55,7 @@ repository = 'https://github.com/pandas-dev/pandas' matplotlib = "pandas:plotting._matplotlib" [project.optional-dependencies] -test = ['hypothesis>=5.5.3', 'pytest>=6.0', 'pytest-xdist>=1.31', 'pytest-asyncio>=0.17.0'] +test = ['hypothesis>=6.34.2', 'pytest>=7.0.0', 'pytest-xdist>=2.2.0', 'pytest-asyncio>=0.17.0'] performance = ['bottleneck>=1.3.2', 'numba>=0.53.1', 'numexpr>=2.7.1'] timezone = ['tzdata>=2022.1'] computation = ['scipy>=1.7.1', 'xarray>=0.21.0'] @@ -87,7 +87,7 @@ all = ['beautifulsoup4>=4.9.3', 'fsspec>=2021.07.0', 'gcsfs>=2021.07.0', 'html5lib>=1.1', - 'hypothesis>=6.13.0', + 'hypothesis>=6.34.2', 'jinja2>=3.0.0', 'lxml>=4.6.3', 'matplotlib>=3.6.1', @@ -101,8 +101,8 @@ all = ['beautifulsoup4>=4.9.3', 'pymysql>=1.0.2', 'PyQt5>=5.15.1', 'pyreadstat>=1.1.2', - 'pytest>=6.0', - 'pytest-xdist>=1.31', + 'pytest>=7.0.0', + 'pytest-xdist>=2.2.0', 'pytest-asyncio>=0.17.0', 'python-snappy>=0.6.0', 'pyxlsb>=1.0.8', @@ -143,7 +143,7 @@ parentdir_prefix = "pandas-" [tool.cibuildwheel] skip = "cp36-* cp37-* pp37-* *-manylinux_i686 *_ppc64le *_s390x *-musllinux*" build-verbosity = "3" -test-requires = "hypothesis==6.52.1 pytest>=6.2.5 pytest-xdist pytest-asyncio>=0.17" +test-requires = "hypothesis>=6.34.2 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17" test-command = "python {project}/ci/test_wheels.py" [tool.cibuildwheel.macos] diff --git a/requirements-dev.txt b/requirements-dev.txt index 4f2a80d932fd0..d06ae4d615a86 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,9 +4,9 @@ pip versioneer[toml] cython==0.29.32 -pytest>=6.0 +pytest>=7.0.0 pytest-cov -pytest-xdist>=1.31 +pytest-xdist>=2.2.0 psutil pytest-asyncio>=0.17 coverage From dbd6a4f8a3362f007214d2ff229a022da8981e1d Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Wed, 28 Dec 2022 17:32:29 -0800 Subject: [PATCH 2/4] forgot one item --- ci/test_wheels_windows.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test_wheels_windows.bat b/ci/test_wheels_windows.bat index 02f2eaf5d2ce7..ae7869d63b1ff 100644 --- a/ci/test_wheels_windows.bat +++ b/ci/test_wheels_windows.bat @@ -4,6 +4,6 @@ pd.test(extra_args=['-m not clipboard and single_cpu', '--skip-slow', '--skip-ne python --version pip install pytz six numpy python-dateutil -pip install hypothesis==6.52.1 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 +pip install hypothesis>=6.34.2 pytest>=7.0.0 pytest-xdist>=2.2.0 pytest-asyncio>=0.17 pip install --find-links=pandas/dist --no-index pandas python -c "%test_command%" From b3961c8478982ea67879295e564af271a8b4bbee Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 3 Jan 2023 18:34:59 -0800 Subject: [PATCH 3/4] Ensure only hypothesis and pytest and checked aligned --- ci/deps/actions-38-minimum_versions.yaml | 2 +- pandas/compat/_optional.py | 2 +- scripts/validate_min_versions_in_sync.py | 23 +++++++++++++---------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/ci/deps/actions-38-minimum_versions.yaml b/ci/deps/actions-38-minimum_versions.yaml index 59050431929e2..caeee07c324d1 100644 --- a/ci/deps/actions-38-minimum_versions.yaml +++ b/ci/deps/actions-38-minimum_versions.yaml @@ -31,7 +31,7 @@ dependencies: - fastparquet=0.6.3 - fsspec=2021.07.0 - html5lib=1.1 - - hypothesis==6.34.2 + - hypothesis=6.34.2 - gcsfs=2021.07.0 - jinja2=3.0.0 - lxml=4.6.3 diff --git a/pandas/compat/_optional.py b/pandas/compat/_optional.py index ce44c80c6fa52..d98b23b215565 100644 --- a/pandas/compat/_optional.py +++ b/pandas/compat/_optional.py @@ -33,7 +33,7 @@ "pymysql": "1.0.2", "pyarrow": "6.0.0", "pyreadstat": "1.1.2", - "pytest": "6.0", + "pytest": "7.0.0", "pyxlsb": "1.0.8", "s3fs": "2021.08.0", "scipy": "1.7.1", diff --git a/scripts/validate_min_versions_in_sync.py b/scripts/validate_min_versions_in_sync.py index 2186e7c8ff9ef..7c102096c1690 100755 --- a/scripts/validate_min_versions_in_sync.py +++ b/scripts/validate_min_versions_in_sync.py @@ -47,11 +47,7 @@ def get_versions_from_code() -> dict[str, str]: versions = _optional.VERSIONS for item in EXCLUDE_DEPS: versions.pop(item, None) - return { - install_map.get(k, k).casefold(): v - for k, v in versions.items() - if k != "pytest" - } + return {install_map.get(k, k).casefold(): v for k, v in versions.items()} def get_versions_from_ci(content: list[str]) -> tuple[dict[str, str], dict[str, str]]: @@ -59,10 +55,18 @@ def get_versions_from_ci(content: list[str]) -> tuple[dict[str, str], dict[str, # Don't parse with pyyaml because it ignores comments we're looking for seen_required = False seen_optional = False + seen_test = False required_deps = {} optional_deps = {} for line in content: - if "# required dependencies" in line: + if "# test dependencies" in line: + seen_test = True + elif seen_test and "- pytest>=" in line: + # Only grab pytest + package, version = line.strip().split(">=") + package = package[2:] + optional_deps[package.casefold()] = version + elif "# required dependencies" in line: seen_required = True elif "# optional dependencies" in line: seen_optional = True @@ -87,7 +91,6 @@ def get_versions_from_ci(content: list[str]) -> tuple[dict[str, str], dict[str, def get_versions_from_toml() -> dict[str, str]: """Min versions in pyproject.toml for pip install pandas[extra].""" install_map = _optional.INSTALL_MAPPING - dependencies = set() optional_dependencies = {} with open(SETUP_PATH, "rb") as pyproject_f: @@ -95,9 +98,9 @@ def get_versions_from_toml() -> dict[str, str]: opt_deps = pyproject_toml["project"]["optional-dependencies"] dependencies = set(opt_deps["all"]) - # remove test dependencies - test_deps = set(opt_deps["test"]) - dependencies = dependencies.difference(test_deps) + # remove pytest plugin dependencies + pytest_plugins = {dep for dep in opt_deps["test"] if dep.startswith("pytest-")} + dependencies = dependencies.difference(pytest_plugins) for dependency in dependencies: package, version = dependency.strip().split(">=") From 8a1a12b639ff7b4bc8f7844e6fd7080da67f5abc Mon Sep 17 00:00:00 2001 From: Matthew Roeschke <10647082+mroeschke@users.noreply.github.com> Date: Tue, 10 Jan 2023 11:32:16 -0800 Subject: [PATCH 4/4] Use conda forge --- ci/deps/actions-310-numpydev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/deps/actions-310-numpydev.yaml b/ci/deps/actions-310-numpydev.yaml index 00db86b09d73e..8b76765e9fba6 100644 --- a/ci/deps/actions-310-numpydev.yaml +++ b/ci/deps/actions-310-numpydev.yaml @@ -1,6 +1,6 @@ name: pandas-dev channels: - - defaults + - conda-forge dependencies: - python=3.10