move int8/16/32 type conversions to index_class_helper.pxi.in

Author: topper-123

doc/source/whatsnew/v0.24.0.txt

@@ -497,15 +497,17 @@ Removal of prior version deprecations/changes
 Performance Improvements
 ~~~~~~~~~~~~~~~~~~~~~~~~
 
-- Very large improvement in performance of slicing when the index is a :class:`CategoricalIndex`,
-  both when indexing by label (using .loc) and position(.iloc).
-  Likewise, slicing a ``CategoricalIndex`` itself (i.e. ``ci[100:200]``) shows similar speed improvements (:issue:`21659`)
-- Improved performance of :func:`Series.describe` in case of numeric dtpyes (:issue:`21274`)
-- Improved performance of :func:`pandas.core.groupby.GroupBy.rank` when dealing with tied rankings (:issue:`21237`)
-- Improved performance of :func:`DataFrame.set_index` with columns consisting of :class:`Period` objects (:issue:`21582`,:issue:`21606`)
+- Slicing Series and Dataframe with an monotonically increasing :class:`CategoricalIndex`
+  is now very fast and has speed comparable to slicing with Int64Index. Memory
+  consumption from such slicing operations is likewise much lower.
+  The speed increase is both when indexing by label (using .loc) and position(.iloc) (:issue:`20395`)
+- Slicing a ``CategoricalIndex`` itself (i.e. ``ci[1000:2000]``) shows similar speed improvements as above (:issue:`21659`)
 - Improved performance of membership checks in :class:`Categorical` and :class:`CategoricalIndex`
   (i.e. ``x in cat``-style checks are much faster). :meth:`CategoricalIndex.contains`
   is likewise much faster (:issue:`21369`, :issue:`21508`)
+- Improved performance of :func:`Series.describe` in case of numeric dtypes (:issue:`21274`)
+- Improved performance of :func:`pandas.core.groupby.GroupBy.rank` when dealing with tied rankings (:issue:`21237`)
+- Improved performance of :func:`DataFrame.set_index` with columns consisting of :class:`Period` objects (:issue:`21582`,:issue:`21606`)
 - Improved performance of :meth:`HDFStore.groups` (and dependent functions like
   :meth:`~HDFStore.keys`.  (i.e. ``x in store`` checks are much faster)
   (:issue:`21372`)

pandas/_libs/algos_common_helper.pxi.in

@@ -22,14 +22,17 @@ WARNING: DO NOT edit .pxi FILE directly, .pxi is generated from .pxi.in
 {{py:
 
 # name, c_type, dtype, can_hold_na, nogil
-dtypes = [('float64', 'float64_t', 'np.float64', True, True),
+dtypes = [('object', 'object', 'object', True, False),
+          ('float64', 'float64_t', 'np.float64', True, True),
           ('float32', 'float32_t', 'np.float32', True, True),
-          ('object', 'object', 'object', True, False),
           ('int64', 'int64_t', 'np.int64', False, True),
-          ('uint64', 'uint64_t', 'np.uint64', False, True),
           ('int32', 'int32_t', 'np.int32', False, True),
           ('int16', 'int16_t', 'np.int16', False, True),
           ('int8', 'int8_t', 'np.int8', False, True),
+          ('uint64', 'uint64_t', 'np.uint64', False, True),
+          ('uint32', 'uint32_t', 'np.uint32', False, True),
+          ('uint16', 'uint16_t', 'np.uint16', False, True),
+          ('uint8', 'uint8_t', 'np.uint8', False, True),
           ('bool', 'uint8_t', 'np.bool', False, True)]
 
 def get_dispatch(dtypes):

pandas/_libs/index.pyx

@@ -8,9 +8,10 @@ from cpython.slice cimport PySlice_Check
 
 import numpy as np
 cimport numpy as cnp
-from numpy cimport (ndarray, float64_t, int32_t,
-                    int8_t, int16_t, int32_t, int64_t,
-                    uint8_t, uint64_t,
+from numpy cimport (ndarray,
+                    float64_t, float32_t,
+                    int64_t,int32_t, int16_t, int8_t,
+                    uint64_t, uint32_t, uint16_t, uint8_t,
                     intp_t,
                     # Note: NPY_DATETIME, NPY_TIMEDELTA are only available
                     # for cimport in cython>=0.27.3
@@ -266,8 +267,6 @@ cdef class IndexEngine:
         if not self.is_mapping_populated:
 
             values = self._get_index_values()
-            if values.dtype in {'int8', 'int16', 'int32'}:
-                values = algos.ensure_int64(values)
             self.mapping = self._make_hash_table(len(values))
             self._call_map_locations(values)
 

pandas/_libs/index_class_helper.pxi.in

@@ -12,11 +12,15 @@ WARNING: DO NOT edit .pxi FILE directly, .pxi is generated from .pxi.in
 
 # name, dtype, ctype
 dtypes = [('Float64', 'float64', 'float64_t'),
+          ('Float32', 'float32', 'float32_t'),
           ('Int64', 'int64', 'int64_t'),
           ('Int32', 'int32', 'int32_t'),
           ('Int16', 'int16', 'int16_t'),
           ('Int8', 'int8', 'int8_t'),
           ('UInt64', 'uint64', 'uint64_t'),
+          ('UInt32', 'uint32', 'uint32_t'),
+          ('UInt16', 'uint16', 'uint16_t'),
+          ('UInt8', 'uint8', 'uint8_t'),
           ('Object', 'object', 'object'),
           ]
 }}
@@ -41,11 +45,24 @@ cdef class {{name}}Engine(IndexEngine):
         {{if name == 'Object'}}
         return _hash.PyObjectHashTable(n)
         {{elif name in {'Int8', 'Int16', 'Int32'} }}
+        # {{name}}HashTable is not available, so we use Int64HashTable
         return _hash.Int64HashTable(n)
+        {{elif name in {'UInt8', 'UInt16', 'UInt32'} }}
+        # {{name}}HashTable is not available, so we use UInt64HashTable
+        return _hash.UInt64HashTable(n)
+        {{elif name in {'Float32'} }}
+        # {{name}}HashTable is not available, so we use Float64HashTable
+        return _hash.Float64HashTable(n)
         {{else}}
         return _hash.{{name}}HashTable(n)
         {{endif}}
 
+    {{if name in {'Int8', 'Int16', 'Int32'} }}
+    cpdef _call_map_locations(self, values):
+        # self.mapping is of type Int64HashTable, so convert dtype of values
+        self.mapping.map_locations(algos.ensure_int64(values))
+    {{endif}}
+
     {{if name != 'Float64' and name != 'Object'}}
     cdef _check_type(self, object val):
         hash(val)

pandas/core/indexes/category.py

@@ -75,8 +75,10 @@ class CategoricalIndex(Index, accessor.PandasDelegate):
 
     @property
     def _engine_type(self):
-        type_name = self.codes.dtype.name.capitalize()
-        return getattr(libindex, "{}Engine".format(type_name))
+        # self.codes can have dtype int8, int16, int 32 or int64, so we need
+        # to return the corresponding engine type (libindex.Int8Engine, etc.).
+        engine_name = "{}Engine".format(self.codes.dtype.name.capitalize())
+        return getattr(libindex, engine_name)
     _attributes = ['name']
 
     def __new__(cls, data=None, categories=None, ordered=None, dtype=None,