deprecate instead of treating as a bug

Author: topper-123

doc/source/whatsnew/v2.1.0.rst

@@ -114,48 +114,10 @@ Notable bug fixes
 
 These are bug fixes that might have notable behavior changes.
 
-.. _whatsnew_210.notable_bug_fixes.series_agg:
-
-User defined functions in Series.agg will always be passed the whole :class:`Series` for evaluation
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-Previously, :meth:`Series.agg` would attempt to apply user-defined functions on each element first and only if that failed would it apply user-defined function to the whole :class:`Series`. This would mean it in some cases didn't aggregate when given an aggregation function, and that the result for :meth:`Series.agg` could be different than the single-column result from :meth:`DataFrame.agg`:
-
-*Previous behavior*:
-
-.. code-block:: ipython
-
-    In [1]: ser = pd.Series([1, 2, 3])
-    In [2]: ser.agg(lambda x: np.sum(x, where=True))
-    Out[2]:
-    0    1
-    1    2
-    2    3
-    dtype: int64
-    In [3]: ser.agg(type)
-    Out[3]:
-    0    <class 'int'>
-    1    <class 'int'>
-    2    <class 'int'>
-    dtype: object
-    In [3]: df = ser.to_frame()
-    In [4]: df.agg(type)[0]
-    pandas.core.series.Series
-
-Now user-defined functions in :meth:`Series.agg` will always be passed the whole :class:`Series` for evaluation:
-
-*New behavior*:
-
-.. ipython:: python
-   :okexcept:
-
-   ser = pd.Series([1, 2, 3])
-   ser.agg(lambda x: np.sum(x, where=True))  # fails, as it should
-   ser.agg(type)
-   ser.agg(type) == ser.to_frame().agg(type)[0]
-
-More generally, the result from :meth:`Series.agg` will now always be the same as the single-column result from :meth:`DataFrame.agg` (:issue:`53325`).
+.. _whatsnew_210.notable_bug_fixes.notable_bug_fix1:
 
+notable_bug_fix1
+^^^^^^^^^^^^^^^^
 
 .. _whatsnew_210.notable_bug_fixes.notable_bug_fix2:
 
@@ -273,6 +235,8 @@ Deprecations
 - Deprecated ``axis=1`` in :meth:`DataFrame.groupby` and in :class:`Grouper` constructor, do ``frame.T.groupby(...)`` instead (:issue:`51203`)
 - Deprecated accepting slices in :meth:`DataFrame.take`, call ``obj[slicer]`` or pass a sequence of integers instead (:issue:`51539`)
 - Deprecated explicit support for subclassing :class:`Index` (:issue:`45289`)
+- Deprecated making functions given to :meth:`Series.agg` attempt to operate on each element in the :class:`Series` and only operate on the whole :class:`Series` if the elementwise operations failed. In the future, functions given to :meth:`Series.agg` will always operate on the whole :class:`Series` only. To keep the current behavior, use :meth:`Series.transform` instead. (:issue:`53325`)
+- Deprecated making the functions in a list of functions given to :meth:`DataFrame.agg` attempt to operate on each element in the :class:`DataFrame` and only operate on the columns of the :class:`DataFrame` if the elementwise operations failed. To keep the current behavior, use :meth:`DataFrame.transform` instead. (:issue:`53325`)
 - Deprecated passing a :class:`DataFrame` to :meth:`DataFrame.from_records`, use :meth:`DataFrame.set_index` or :meth:`DataFrame.drop` instead (:issue:`51353`)
 - Deprecated silently dropping unrecognized timezones when parsing strings to datetimes (:issue:`18702`)
 - Deprecated the ``axis`` keyword in :meth:`DataFrame.ewm`, :meth:`Series.ewm`, :meth:`DataFrame.rolling`, :meth:`Series.rolling`, :meth:`DataFrame.expanding`, :meth:`Series.expanding` (:issue:`51778`)

pandas/core/apply.py

@@ -1121,23 +1121,28 @@ def apply(self) -> DataFrame | Series:
     def agg(self):
         result = super().agg()
         if result is None:
+            obj = self.obj
             func = self.func
-
             # string, list-like, and dict-like are entirely handled in super
             assert callable(func)
 
-            # try a regular apply, this evaluates lambdas
-            # row-by-row; however if the lambda is expected a Series
-            # expression, e.g.: lambda x: x-x.quantile(0.25)
-            # this will fail, so we can try a vectorized evaluation
-
-            # we cannot FIRST try the vectorized evaluation, because
-            # then .agg and .apply would have different semantics if the
-            # operation is actually defined on the Series, e.g. str
+            # GH53325: The setup below is just to keep current behavior while emitting a
+            # deprecation message. In the future this will all be replaced with a simple
+            # `result = f(self.obj)`.
+            if isinstance(func, np.ufunc):
+                with np.errstate(all="ignore"):
+                    return func(obj)
             try:
-                result = self.obj.apply(func, args=self.args, **self.kwargs)
+                result = obj.apply(func)
             except (ValueError, AttributeError, TypeError):
-                result = func(self.obj, *self.args, **self.kwargs)
+                result = func(self.obj)
+            else:
+                msg = (
+                    f"using {func} in {type(obj).__name__}.agg cannot aggregate and "
+                    f"has been deprecated. Use {type(obj).__name__}.transform to "
+                    f"keep behavior unchanged."
+                )
+                warnings.warn(msg, FutureWarning, stacklevel=find_stack_level())
 
         return result
 

pandas/tests/apply/test_frame_apply.py

@@ -1478,8 +1478,8 @@ def test_any_apply_keyword_non_zero_axis_regression():
     tm.assert_series_equal(result, expected)
 
 
-def test_agg_list_like_func_with_args():
-    # GH 50624
+def test_agg_mapping_func_deprecated():
+    # GH 53325
     df = DataFrame({"x": [1, 2, 3]})
 
     def foo1(x, a=1, c=0):
@@ -1488,16 +1488,13 @@ def foo1(x, a=1, c=0):
     def foo2(x, b=2, c=0):
         return x + b + c
 
-    msg = r"foo1\(\) got an unexpected keyword argument 'b'"
-    with pytest.raises(TypeError, match=msg):
-        df.agg([foo1, foo2], 0, 3, b=3, c=4)
-
-    result = df.agg([foo1, foo2], 0, 3, c=4)
-    expected = DataFrame(
-        [[8, 8], [9, 9], [10, 10]],
-        columns=MultiIndex.from_tuples([("x", "foo1"), ("x", "foo2")]),
-    )
-    tm.assert_frame_equal(result, expected)
+    # single func already takes the vectorized path
+    df.agg(foo1, 0, 3, c=4)
+    msg = "using .+ in Series.agg cannot aggregate and"
+    with tm.assert_produces_warning(FutureWarning, match=msg):
+        df.agg([foo1, foo2], 0, 3, c=4)
+        with tm.assert_produces_warning(FutureWarning, match=msg):
+            df.agg({"x": foo1}, 0, 3, c=4)
 
 
 def test_agg_std():

pandas/tests/apply/test_frame_transform.py

@@ -66,6 +66,28 @@ def test_transform_empty_listlike(float_frame, ops, frame_or_series):
         obj.transform(ops)
 
 
+def test_transform_listlike_func_with_args():
+    # GH 50624
+    df = DataFrame({"x": [1, 2, 3]})
+
+    def foo1(x, a=1, c=0):
+        return x + a + c
+
+    def foo2(x, b=2, c=0):
+        return x + b + c
+
+    msg = r"foo1\(\) got an unexpected keyword argument 'b'"
+    with pytest.raises(TypeError, match=msg):
+        df.transform([foo1, foo2], 0, 3, b=3, c=4)
+
+    result = df.transform([foo1, foo2], 0, 3, c=4)
+    expected = DataFrame(
+        [[8, 8], [9, 9], [10, 10]],
+        columns=MultiIndex.from_tuples([("x", "foo1"), ("x", "foo2")]),
+    )
+    tm.assert_frame_equal(result, expected)
+
+
 @pytest.mark.parametrize("box", [dict, Series])
 def test_transform_dictlike(axis, float_frame, box):
     # GH 35964

pandas/tests/apply/test_series_apply.py

@@ -98,24 +98,18 @@ def test_apply_args():
     assert isinstance(result[0], list)
 
 
-@pytest.mark.parametrize(
-    "args, kwargs, increment",
-    [((), {}, 0), ((), {"a": 1}, 1), ((2, 3), {}, 32), ((1,), {"c": 2}, 201)],
-)
-def test_agg_args(args, kwargs, increment):
-    # GH 43357
-    def f(x, a=0, b=0, c=0):
-        return x + a + 10 * b + 100 * c
+def test_agg_args():
+    def f(x, increment):
+        return x.sum() + increment
 
     s = Series([1, 2])
-    result = s.agg(f, 0, *args, **kwargs)
-    expected = s + increment
-    tm.assert_series_equal(result, expected)
-
+    result = s.agg(f, increment=0)
+    expected = s.sum()
+    assert result == expected
 
-def test_agg_list_like_func_with_args():
-    # GH 50624
 
+def test_agg_mapping_func_deprecated():
+    # GH 53325
     s = Series([1, 2, 3])
 
     def foo1(x, a=1, c=0):
@@ -124,13 +118,13 @@ def foo1(x, a=1, c=0):
     def foo2(x, b=2, c=0):
         return x + b + c
 
-    msg = r"foo1\(\) got an unexpected keyword argument 'b'"
-    with pytest.raises(TypeError, match=msg):
-        s.agg([foo1, foo2], 0, 3, b=3, c=4)
-
-    result = s.agg([foo1, foo2], 0, 3, c=4)
-    expected = DataFrame({"foo1": [8, 9, 10], "foo2": [8, 9, 10]})
-    tm.assert_frame_equal(result, expected)
+    msg = "using .+ in Series.agg cannot aggregate and"
+    with tm.assert_produces_warning(FutureWarning, match=msg):
+        s.agg(foo1, 0, 3, c=4)
+    with tm.assert_produces_warning(FutureWarning, match=msg):
+        s.agg([foo1, foo2], 0, 3, c=4)
+    with tm.assert_produces_warning(FutureWarning, match=msg):
+        s.agg({"a": foo1, "b": foo2}, 0, 3, c=4)
 
 
 def test_series_apply_map_box_timestamps(by_row):
@@ -393,13 +387,15 @@ def test_apply_map_evaluate_lambdas_the_same(string_series, func, by_row):
 
 def test_agg_evaluate_lambdas(string_series):
     # GH53325
-    expected = Series
+    # in the future, the result will be a Series class.
 
-    result = string_series.agg(lambda x: type(x))
-    assert result is expected
+    with tm.assert_produces_warning(FutureWarning):
+        result = string_series.agg(lambda x: type(x))
+    assert isinstance(result, Series) and len(result) == len(string_series)
 
-    result = string_series.agg(type)
-    assert result is expected
+    with tm.assert_produces_warning(FutureWarning):
+        result = string_series.agg(type)
+    assert isinstance(result, Series) and len(result) == len(string_series)
 
 
 def test_with_nested_series(datetime_series):

pandas/tests/apply/test_series_transform.py

@@ -10,6 +10,21 @@
 import pandas._testing as tm
 
 
+@pytest.mark.parametrize(
+    "args, kwargs, increment",
+    [((), {}, 0), ((), {"a": 1}, 1), ((2, 3), {}, 32), ((1,), {"c": 2}, 201)],
+)
+def test_agg_args(args, kwargs, increment):
+    # GH 43357
+    def f(x, a=0, b=0, c=0):
+        return x + a + 10 * b + 100 * c
+
+    s = Series([1, 2])
+    result = s.transform(f, 0, *args, **kwargs)
+    expected = s + increment
+    tm.assert_series_equal(result, expected)
+
+
 @pytest.mark.parametrize(
     "ops, names",
     [
@@ -28,6 +43,26 @@ def test_transform_listlike(string_series, ops, names):
         tm.assert_frame_equal(result, expected)
 
 
+def test_transform_listlike_func_with_args():
+    # GH 50624
+
+    s = Series([1, 2, 3])
+
+    def foo1(x, a=1, c=0):
+        return x + a + c
+
+    def foo2(x, b=2, c=0):
+        return x + b + c
+
+    msg = r"foo1\(\) got an unexpected keyword argument 'b'"
+    with pytest.raises(TypeError, match=msg):
+        s.transform([foo1, foo2], 0, 3, b=3, c=4)
+
+    result = s.transform([foo1, foo2], 0, 3, c=4)
+    expected = DataFrame({"foo1": [8, 9, 10], "foo2": [8, 9, 10]})
+    tm.assert_frame_equal(result, expected)
+
+
 @pytest.mark.parametrize("box", [dict, Series])
 def test_transform_dictlike(string_series, box):
     # GH 35964