From f8d20e52828bec86f2c14772af19b9330efcc81b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Thu, 9 Mar 2023 13:19:21 -0500 Subject: [PATCH 1/5] TYP: fix NDFrame.align --- .pre-commit-config.yaml | 2 +- pandas/core/frame.py | 12 ++++++------ pandas/core/generic.py | 14 +++++++------- pandas/core/series.py | 9 ++++++--- pyright_reportGeneralTypeIssues.json | 6 ++---- 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f10cfac5f1276..9051b66d14a30 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -132,7 +132,7 @@ repos: types: [python] stages: [manual] additional_dependencies: &pyright_dependencies - - pyright@1.1.292 + - pyright@1.1.298 - id: pyright_reportGeneralTypeIssues # note: assumes python env is setup and activated name: pyright reportGeneralTypeIssues diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 3a48c3b88e071..15ad7a14be3ed 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -4997,7 +4997,7 @@ def _reindex_multi( @doc(NDFrame.align, **_shared_doc_kwargs) def align( self, - other: DataFrame, + other: NDFrame, join: AlignJoin = "outer", axis: Axis | None = None, level: Level = None, @@ -5007,8 +5007,10 @@ def align( limit: int | None = None, fill_axis: Axis = 0, broadcast_axis: Axis | None = None, - ) -> DataFrame: - return super().align( + ) -> tuple[DataFrame, DataFrame]: + # error: Incompatible return value type (got "Tuple[NDFrame, NDFrame]", + # expected "Tuple[DataFrame, DataFrame]") + return super().align( # type: ignore[return-value] other, join=join, axis=axis, @@ -7771,9 +7773,7 @@ def to_series(right): ) left, right = left.align( - # error: Argument 1 to "align" of "DataFrame" has incompatible - # type "Series"; expected "DataFrame" - right, # type: ignore[arg-type] + right, join="outer", axis=axis, level=level, diff --git a/pandas/core/generic.py b/pandas/core/generic.py index a4dfb085c766f..0581880a4007c 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -9296,8 +9296,8 @@ def compare( @doc(**_shared_doc_kwargs) def align( - self: NDFrameT, - other: NDFrameT, + self, + other: NDFrame, join: AlignJoin = "outer", axis: Axis | None = None, level: Level = None, @@ -9307,7 +9307,7 @@ def align( limit: int | None = None, fill_axis: Axis = 0, broadcast_axis: Axis | None = None, - ) -> NDFrameT: + ) -> tuple[NDFrame, NDFrame]: """ Align two objects on their axes with the specified join method. @@ -9489,7 +9489,7 @@ def align( @final def _align_frame( - self, + self: NDFrameT, other, join: AlignJoin = "outer", axis: Axis | None = None, @@ -9499,7 +9499,7 @@ def _align_frame( method=None, limit=None, fill_axis: Axis = 0, - ): + ) -> tuple[NDFrameT, NDFrameT]: # defaults join_index, join_columns = None, None ilidx, iridx = None, None @@ -9553,7 +9553,7 @@ def _align_frame( @final def _align_series( - self, + self: NDFrameT, other, join: AlignJoin = "outer", axis: Axis | None = None, @@ -9563,7 +9563,7 @@ def _align_series( method=None, limit=None, fill_axis: Axis = 0, - ): + ) -> tuple[NDFrameT, NDFrameT]: is_series = isinstance(self, ABCSeries) if copy and using_copy_on_write(): copy = False diff --git a/pandas/core/series.py b/pandas/core/series.py index 25cb18b0135fa..ae468106fe347 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -166,6 +166,7 @@ IndexLabel, Level, NaPosition, + NDFrameT, NumpySorter, NumpyValueArrayLike, QuantileInterpolation, @@ -4561,7 +4562,7 @@ def _needs_reindex_multi(self, axes, method, level) -> bool: ) def align( self, - other: Series, + other: NDFrameT, join: AlignJoin = "outer", axis: Axis | None = None, level: Level = None, @@ -4571,8 +4572,10 @@ def align( limit: int | None = None, fill_axis: Axis = 0, broadcast_axis: Axis | None = None, - ) -> Series: - return super().align( + ) -> tuple[NDFrameT, NDFrameT]: + # error: Incompatible return value type (got "Tuple[NDFrame, NDFrame]", + # expected "Tuple[NDFrameT, NDFrameT]") + return super().align( # type: ignore[return-value] other, join=join, axis=axis, diff --git a/pyright_reportGeneralTypeIssues.json b/pyright_reportGeneralTypeIssues.json index 761018c3ce496..b5baab8c33471 100644 --- a/pyright_reportGeneralTypeIssues.json +++ b/pyright_reportGeneralTypeIssues.json @@ -1,4 +1,3 @@ -# this becomes obsolete when reportGeneralTypeIssues can be enabled in pyproject.toml { "typeCheckingMode": "off", "reportGeneralTypeIssues": true, @@ -9,12 +8,11 @@ ], "exclude": [ - # exclude tests "pandas/tests", - # exclude vendored files + "pandas/io/clipboard", "pandas/util/version", - # and all files that currently don't pass + "pandas/_testing/__init__.py", "pandas/_testing/_hypothesis.py", "pandas/_testing/_io.py", From 0178cb6694ed332ddd25a36b90dbf8ed9f93a5b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Thu, 9 Mar 2023 14:22:43 -0500 Subject: [PATCH 2/5] avoid ignores outside NDFrame.align (but add them inside NDFrame.align) --- pandas/core/frame.py | 9 ++++----- pandas/core/generic.py | 39 +++++++++++++++++++++++++-------------- pandas/core/series.py | 6 ++---- 3 files changed, 31 insertions(+), 23 deletions(-) diff --git a/pandas/core/frame.py b/pandas/core/frame.py index 15ad7a14be3ed..70019030da182 100644 --- a/pandas/core/frame.py +++ b/pandas/core/frame.py @@ -225,6 +225,7 @@ Level, MergeHow, NaPosition, + NDFrameT, PythonFuncType, QuantileInterpolation, ReadBuffer, @@ -4997,7 +4998,7 @@ def _reindex_multi( @doc(NDFrame.align, **_shared_doc_kwargs) def align( self, - other: NDFrame, + other: NDFrameT, join: AlignJoin = "outer", axis: Axis | None = None, level: Level = None, @@ -5007,10 +5008,8 @@ def align( limit: int | None = None, fill_axis: Axis = 0, broadcast_axis: Axis | None = None, - ) -> tuple[DataFrame, DataFrame]: - # error: Incompatible return value type (got "Tuple[NDFrame, NDFrame]", - # expected "Tuple[DataFrame, DataFrame]") - return super().align( # type: ignore[return-value] + ) -> tuple[DataFrame, NDFrameT]: + return super().align( other, join=join, axis=axis, diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 0581880a4007c..8ec23bf2e7abc 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -21,6 +21,7 @@ NoReturn, Sequence, Type, + TypeVar, cast, final, overload, @@ -198,6 +199,8 @@ from pandas.core.indexers.objects import BaseIndexer from pandas.core.resample import Resampler + _NDFrameT = TypeVar("_NDFrameT", bound="NDFrame") + # goal is to be able to define the docs close to function, while still being # able to share @@ -9296,8 +9299,8 @@ def compare( @doc(**_shared_doc_kwargs) def align( - self, - other: NDFrame, + self: NDFrameT, + other: _NDFrameT, join: AlignJoin = "outer", axis: Axis | None = None, level: Level = None, @@ -9307,7 +9310,7 @@ def align( limit: int | None = None, fill_axis: Axis = 0, broadcast_axis: Axis | None = None, - ) -> tuple[NDFrame, NDFrame]: + ) -> tuple[NDFrameT, _NDFrameT]: """ Align two objects on their axes with the specified join method. @@ -9428,8 +9431,10 @@ def align( df = cons( {c: self for c in other.columns}, **other._construct_axes_dict() ) - return df._align_frame( - other, + # error: Incompatible return value type (got "Tuple[DataFrame, + # DataFrame]", expected "Tuple[NDFrameT, _NDFrameT]") + return df._align_frame( # type: ignore[return-value] + other, # type: ignore[arg-type] join=join, axis=axis, level=level, @@ -9446,7 +9451,9 @@ def align( df = cons( {c: other for c in self.columns}, **self._construct_axes_dict() ) - return self._align_frame( + # error: Incompatible return value type (got "Tuple[NDFrameT, + # DataFrame]", expected "Tuple[NDFrameT, _NDFrameT]") + return self._align_frame( # type: ignore[return-value] df, join=join, axis=axis, @@ -9461,7 +9468,9 @@ def align( if axis is not None: axis = self._get_axis_number(axis) if isinstance(other, ABCDataFrame): - return self._align_frame( + # error: Incompatible return value type (got "Tuple[NDFrameT, DataFrame]", + # expected "Tuple[NDFrameT, _NDFrameT]") + return self._align_frame( # type: ignore[return-value] other, join=join, axis=axis, @@ -9473,7 +9482,9 @@ def align( fill_axis=fill_axis, ) elif isinstance(other, ABCSeries): - return self._align_series( + # error: Incompatible return value type (got "Tuple[NDFrameT, Series]", + # expected "Tuple[NDFrameT, _NDFrameT]") + return self._align_series( # type: ignore[return-value] other, join=join, axis=axis, @@ -9490,7 +9501,7 @@ def align( @final def _align_frame( self: NDFrameT, - other, + other: DataFrame, join: AlignJoin = "outer", axis: Axis | None = None, level=None, @@ -9499,7 +9510,7 @@ def _align_frame( method=None, limit=None, fill_axis: Axis = 0, - ) -> tuple[NDFrameT, NDFrameT]: + ) -> tuple[NDFrameT, DataFrame]: # defaults join_index, join_columns = None, None ilidx, iridx = None, None @@ -9554,7 +9565,7 @@ def _align_frame( @final def _align_series( self: NDFrameT, - other, + other: Series, join: AlignJoin = "outer", axis: Axis | None = None, level=None, @@ -9563,7 +9574,7 @@ def _align_series( method=None, limit=None, fill_axis: Axis = 0, - ) -> tuple[NDFrameT, NDFrameT]: + ) -> tuple[NDFrameT, Series]: is_series = isinstance(self, ABCSeries) if copy and using_copy_on_write(): copy = False @@ -12798,8 +12809,8 @@ def _doc_params(cls): def _align_as_utc( - left: NDFrameT, right: NDFrameT, join_index: Index | None -) -> tuple[NDFrameT, NDFrameT]: + left: NDFrameT, right: _NDFrameT, join_index: Index | None +) -> tuple[NDFrameT, _NDFrameT]: """ If we are aligning timezone-aware DatetimeIndexes and the timezones do not match, convert both to UTC. diff --git a/pandas/core/series.py b/pandas/core/series.py index ae468106fe347..5de760c4ed82c 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -4572,10 +4572,8 @@ def align( limit: int | None = None, fill_axis: Axis = 0, broadcast_axis: Axis | None = None, - ) -> tuple[NDFrameT, NDFrameT]: - # error: Incompatible return value type (got "Tuple[NDFrame, NDFrame]", - # expected "Tuple[NDFrameT, NDFrameT]") - return super().align( # type: ignore[return-value] + ) -> tuple[Series, NDFrameT]: + return super().align( other, join=join, axis=axis, From 870709139fa4a2bdc5d8d2586ea9aa29b3384c9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Thu, 9 Mar 2023 18:12:02 -0500 Subject: [PATCH 3/5] added comment --- pandas/core/generic.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 8ec23bf2e7abc..5fb30e76c1e68 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -199,6 +199,9 @@ from pandas.core.indexers.objects import BaseIndexer from pandas.core.resample import Resampler + # Needed when having two different pairs of variables that should be + # allowed to have different NDFrame sub-classes (see _align_as_utc + # and align). _NDFrameT = TypeVar("_NDFrameT", bound="NDFrame") From 702255771c488d483bbc414bc737f63910dba25b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Fri, 10 Mar 2023 10:28:36 -0500 Subject: [PATCH 4/5] move to pandas._typing --- pandas/_typing.py | 3 +++ pandas/core/generic.py | 24 +++++++++--------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/pandas/_typing.py b/pandas/_typing.py index 6059bced4a7d4..b54b8d6adb50e 100644 --- a/pandas/_typing.py +++ b/pandas/_typing.py @@ -125,6 +125,9 @@ # Series is passed into a function, a Series is always returned and if a DataFrame is # passed in, a DataFrame is always returned. NDFrameT = TypeVar("NDFrameT", bound="NDFrame") +# same as NDFrameT, needed when binding two pairs of parameters to potentially +# separate NDFrame-subclasses (see NDFrame.align) +NDFrameTb = TypeVar("NDFrameTb", bound="NDFrame") NumpyIndexT = TypeVar("NumpyIndexT", np.ndarray, "Index") diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 5fb30e76c1e68..95ac522833b35 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -21,7 +21,6 @@ NoReturn, Sequence, Type, - TypeVar, cast, final, overload, @@ -68,6 +67,7 @@ Manager, NaPosition, NDFrameT, + NDFrameTb, RandomState, Renamer, Scalar, @@ -199,12 +199,6 @@ from pandas.core.indexers.objects import BaseIndexer from pandas.core.resample import Resampler - # Needed when having two different pairs of variables that should be - # allowed to have different NDFrame sub-classes (see _align_as_utc - # and align). - _NDFrameT = TypeVar("_NDFrameT", bound="NDFrame") - - # goal is to be able to define the docs close to function, while still being # able to share _shared_docs = {**_shared_docs} @@ -9303,7 +9297,7 @@ def compare( @doc(**_shared_doc_kwargs) def align( self: NDFrameT, - other: _NDFrameT, + other: NDFrameTb, join: AlignJoin = "outer", axis: Axis | None = None, level: Level = None, @@ -9313,7 +9307,7 @@ def align( limit: int | None = None, fill_axis: Axis = 0, broadcast_axis: Axis | None = None, - ) -> tuple[NDFrameT, _NDFrameT]: + ) -> tuple[NDFrameT, NDFrameTb]: """ Align two objects on their axes with the specified join method. @@ -9435,7 +9429,7 @@ def align( {c: self for c in other.columns}, **other._construct_axes_dict() ) # error: Incompatible return value type (got "Tuple[DataFrame, - # DataFrame]", expected "Tuple[NDFrameT, _NDFrameT]") + # DataFrame]", expected "Tuple[NDFrameT, NDFrameTb]") return df._align_frame( # type: ignore[return-value] other, # type: ignore[arg-type] join=join, @@ -9455,7 +9449,7 @@ def align( {c: other for c in self.columns}, **self._construct_axes_dict() ) # error: Incompatible return value type (got "Tuple[NDFrameT, - # DataFrame]", expected "Tuple[NDFrameT, _NDFrameT]") + # DataFrame]", expected "Tuple[NDFrameT, NDFrameTb]") return self._align_frame( # type: ignore[return-value] df, join=join, @@ -9472,7 +9466,7 @@ def align( axis = self._get_axis_number(axis) if isinstance(other, ABCDataFrame): # error: Incompatible return value type (got "Tuple[NDFrameT, DataFrame]", - # expected "Tuple[NDFrameT, _NDFrameT]") + # expected "Tuple[NDFrameT, NDFrameTb]") return self._align_frame( # type: ignore[return-value] other, join=join, @@ -9486,7 +9480,7 @@ def align( ) elif isinstance(other, ABCSeries): # error: Incompatible return value type (got "Tuple[NDFrameT, Series]", - # expected "Tuple[NDFrameT, _NDFrameT]") + # expected "Tuple[NDFrameT, NDFrameTb]") return self._align_series( # type: ignore[return-value] other, join=join, @@ -12812,8 +12806,8 @@ def _doc_params(cls): def _align_as_utc( - left: NDFrameT, right: _NDFrameT, join_index: Index | None -) -> tuple[NDFrameT, _NDFrameT]: + left: NDFrameT, right: NDFrameTb, join_index: Index | None +) -> tuple[NDFrameT, NDFrameTb]: """ If we are aligning timezone-aware DatetimeIndexes and the timezones do not match, convert both to UTC. From f42c2adb8b29ed9079ea0cf23007fc916d7ad72e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torsten=20W=C3=B6rtwein?= Date: Sat, 11 Mar 2023 14:03:49 -0500 Subject: [PATCH 5/5] undo pyright change --- .pre-commit-config.yaml | 2 +- pyright_reportGeneralTypeIssues.json | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9051b66d14a30..f10cfac5f1276 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -132,7 +132,7 @@ repos: types: [python] stages: [manual] additional_dependencies: &pyright_dependencies - - pyright@1.1.298 + - pyright@1.1.292 - id: pyright_reportGeneralTypeIssues # note: assumes python env is setup and activated name: pyright reportGeneralTypeIssues diff --git a/pyright_reportGeneralTypeIssues.json b/pyright_reportGeneralTypeIssues.json index b5baab8c33471..761018c3ce496 100644 --- a/pyright_reportGeneralTypeIssues.json +++ b/pyright_reportGeneralTypeIssues.json @@ -1,3 +1,4 @@ +# this becomes obsolete when reportGeneralTypeIssues can be enabled in pyproject.toml { "typeCheckingMode": "off", "reportGeneralTypeIssues": true, @@ -8,11 +9,12 @@ ], "exclude": [ + # exclude tests "pandas/tests", - + # exclude vendored files "pandas/io/clipboard", "pandas/util/version", - + # and all files that currently don't pass "pandas/_testing/__init__.py", "pandas/_testing/_hypothesis.py", "pandas/_testing/_io.py",