From fba63e290fa74d5dd70c993981fe731771c953d8 Mon Sep 17 00:00:00 2001 From: ischwabacher Date: Thu, 3 Jul 2014 16:36:32 -0500 Subject: [PATCH 1/5] FIX: Scalar timedelta NaT results raise Currently, coercion of scalar results from `float` to `timedelta64[ns]` passes through `int`, which raises when attempting to coerce `NaN` to `NaT`. To reproduce: ```python import pandas as pd import numpy as np pd.Series([np.timedelta64('NaT')]).sum() # TypeError: reduction operation 'sum' not allowed for this dtype ``` --- pandas/core/nanops.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pandas/core/nanops.py b/pandas/core/nanops.py index aa6140383a27a..b38bc9142f7f5 100644 --- a/pandas/core/nanops.py +++ b/pandas/core/nanops.py @@ -230,7 +230,9 @@ def _wrap_results(result, dtype): from pandas import Series # coerce float to results - if is_float(result): + if isnull(result): + result = tslib.NaT + elif is_float(result): result = int(result) result = Series([result], dtype='timedelta64[ns]') else: From 03d1aa9cd914119526ca130c5a7a2f9b9af67b2a Mon Sep 17 00:00:00 2001 From: ischwabacher Date: Thu, 3 Jul 2014 17:41:42 -0500 Subject: [PATCH 2/5] TST: Add tests for datetime and timedelta NaT Test coverage for ops on `NaT` values is a little better. The SciPy functions for which operations on `datetime64` and `timedelta64` make sense could be added, but to my knowledge SciPy will currently choke on these anyway. --- pandas/tests/test_nanops.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index 3e8a5fecbb579..94173d34eed69 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -30,6 +30,8 @@ def setUp(self): self.arr_shape).astype('m8[ns]') self.arr_nan = np.tile(np.nan, self.arr_shape) + self.arr_datenat = np.tile(np.datetime64('NaT'), self.arr_shape) + self.arr_tdeltanat = np.tile(np.timedelta64('NaT'), self.arr_shape) self.arr_float_nan = np.vstack([self.arr_float, self.arr_nan]) self.arr_float1_nan = np.vstack([self.arr_float1, self.arr_nan]) self.arr_nan_float1 = np.vstack([self.arr_nan, self.arr_float1]) @@ -244,13 +246,18 @@ def check_funs(self, testfunc, targfunc, else: self.check_fun(testfunc, targfunc, 'arr_date', **kwargs) objs += [self.arr_date.astype('O')] + if allow_all_nan: + self.check_fun(testfunc, targfunc, 'arr_datenat', + **kwargs) try: targfunc(self.arr_tdelta) except TypeError: pass else: self.check_fun(testfunc, targfunc, 'arr_tdelta', **kwargs) - objs += [self.arr_tdelta.astype('O')] + if allow_all_nan: + self.check_fun(testfunc, targfunc, 'arr_tdeltanat', + **kwargs) if allow_obj: self.arr_obj = np.vstack(objs) @@ -291,18 +298,15 @@ def test_nanall(self): allow_all_nan=False, allow_str=False, allow_date=False) def test_nansum(self): - self.check_funs(nanops.nansum, np.sum, - allow_str=False, allow_date=False) + self.check_funs(nanops.nansum, np.sum, allow_str=False) def test_nanmean(self): - self.check_funs(nanops.nanmean, np.mean, - allow_complex=False, allow_obj=False, - allow_str=False, allow_date=False) + self.check_funs(nanops.nanmean, np.mean, allow_complex=False, + allow_obj=False, allow_str=False) def test_nanmedian(self): - self.check_funs(nanops.nanmedian, np.median, - allow_complex=False, allow_str=False, allow_date=False, - allow_obj='convert') + self.check_funs(nanops.nanmedian, np.median, allow_complex=False, + allow_str=False, allow_obj='convert') def test_nanvar(self): self.check_funs_ddof(nanops.nanvar, np.var, @@ -349,7 +353,6 @@ def test_nanargmin(self): func = partial(self._argminmax_wrap, func=np.argmin) if tm.sys.version_info[0:2] == (2, 6): self.check_funs(nanops.nanargmin, func, - allow_date=False, allow_str=False, allow_obj=False) else: self.check_funs(nanops.nanargmin, func, From 5d33112db243206cb437757f2154a64735e0db94 Mon Sep 17 00:00:00 2001 From: ischwabacher Date: Thu, 3 Jul 2014 17:43:52 -0500 Subject: [PATCH 3/5] TST: Revert stupidity in previous commit. --- pandas/tests/test_nanops.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index 94173d34eed69..7f691089325e5 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -255,6 +255,7 @@ def check_funs(self, testfunc, targfunc, pass else: self.check_fun(testfunc, targfunc, 'arr_tdelta', **kwargs) + objs += [self.arr_date.astype('O')] if allow_all_nan: self.check_fun(testfunc, targfunc, 'arr_tdeltanat', **kwargs) From 6957cac6ca33d96d03ede1279d6ee0b544f550df Mon Sep 17 00:00:00 2001 From: ischwabacher Date: Thu, 3 Jul 2014 18:14:43 -0500 Subject: [PATCH 4/5] Use NaTs of type `m8[ns]` instead of `m8` --- pandas/tests/test_nanops.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index 7f691089325e5..9325a4a872821 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -30,8 +30,9 @@ def setUp(self): self.arr_shape).astype('m8[ns]') self.arr_nan = np.tile(np.nan, self.arr_shape) - self.arr_datenat = np.tile(np.datetime64('NaT'), self.arr_shape) - self.arr_tdeltanat = np.tile(np.timedelta64('NaT'), self.arr_shape) + self.arr_datenat = np.tile(np.datetime64('NaT', 'ns'), self.arr_shape) + self.arr_tdeltanat = np.tile(np.timedelta64('NaT', 'ns'), + self.arr_shape) self.arr_float_nan = np.vstack([self.arr_float, self.arr_nan]) self.arr_float1_nan = np.vstack([self.arr_float1, self.arr_nan]) self.arr_nan_float1 = np.vstack([self.arr_nan, self.arr_float1]) From 62370ace6d4e7969e2bd913c995a9b3eab8cd11d Mon Sep 17 00:00:00 2001 From: ischwabacher Date: Mon, 7 Jul 2014 10:33:06 -0500 Subject: [PATCH 5/5] Construct test cases with iNaT instead of 'NaT' Note that `print(np.array([pd.tslib.iNaT], dtype='m8[ns]'))` does not display correctly, but its behavior is consistent with being Not-A-Time. --- pandas/tests/test_nanops.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/tests/test_nanops.py b/pandas/tests/test_nanops.py index 9325a4a872821..483a07264482c 100644 --- a/pandas/tests/test_nanops.py +++ b/pandas/tests/test_nanops.py @@ -5,6 +5,7 @@ import numpy as np from pandas.core.common import isnull +from pandas.tslib import iNaT import pandas.core.nanops as nanops import pandas.util.testing as tm @@ -30,9 +31,8 @@ def setUp(self): self.arr_shape).astype('m8[ns]') self.arr_nan = np.tile(np.nan, self.arr_shape) - self.arr_datenat = np.tile(np.datetime64('NaT', 'ns'), self.arr_shape) - self.arr_tdeltanat = np.tile(np.timedelta64('NaT', 'ns'), - self.arr_shape) + self.arr_datenat = np.tile(iNaT, self.arr_shape).astype('M8[ns]') + self.arr_tdeltanat = np.tile(iNaT, self.arr_shape).astype('m8[ns]') self.arr_float_nan = np.vstack([self.arr_float, self.arr_nan]) self.arr_float1_nan = np.vstack([self.arr_float1, self.arr_nan]) self.arr_nan_float1 = np.vstack([self.arr_nan, self.arr_float1])