From 96b9080bb17b6c4fa5dd2687bda7702434b4f84c Mon Sep 17 00:00:00 2001 From: gnzlbg Date: Sat, 21 Oct 2017 18:28:35 +0200 Subject: [PATCH 01/12] [vec] growth-strategy optimization --- src/liballoc/raw_vec.rs | 243 ++++++++++++++++++++++++++------------ src/liballoc/vec.rs | 18 ++- src/liballoc/vec_deque.rs | 5 +- 3 files changed, 180 insertions(+), 86 deletions(-) diff --git a/src/liballoc/raw_vec.rs b/src/liballoc/raw_vec.rs index dbf1fb1367dda..8d02e916ddf5b 100644 --- a/src/liballoc/raw_vec.rs +++ b/src/liballoc/raw_vec.rs @@ -235,7 +235,60 @@ impl RawVec { } } - /// Doubles the size of the type's backing allocation. This is common enough + /// Grows the vector capacity by `capacity_increase`. + /// + /// It allows implementing amortized O(1) `push` on vector-like containers. + /// + /// # Attributes + /// + /// - `#[inline]`: LLVM is able to completely elide memory allocations in + /// many cases if it can "see" where memory is allocated and freed. + /// - `#[cold]`: calling this function is a "rare" event + /// + #[inline] + #[cold] + pub fn grow_by(&mut self, capacity_increase: usize) { + let elem_size = mem::size_of::(); + assert!(elem_size != 0, "RawVecs of zero-sized types can't grow"); + + let (new_cap, uniq) = match self.current_layout() { + Some(cur) => { + // The invariant `elem_size * self.cap <= isize::MAX` is + // maintained by `alloc_guard`; the alignment will never be too + // large as to "not be specifiable" (so we can use + // `from_size_align_unchecked`). + let new_cap = Self::suitable_capacity(self.cap, capacity_increase); + let new_size = new_cap * elem_size; + let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, cur.align()) }; + let (_, usable_size) = self.a.usable_size(&new_layout); + let new_layout = unsafe { Layout::from_size_align_unchecked(usable_size, cur.align()) }; + alloc_guard(usable_size); + let ptr_res = unsafe { self.a.realloc(self.ptr.as_ptr() as *mut u8, + cur, + new_layout) }; + match ptr_res { + Ok(ptr) => (new_cap, unsafe { Unique::new_unchecked(ptr as *mut T) }), + Err(e) => self.a.oom(e), + } + } + None => { + let new_cap = Self::suitable_capacity(self.cap, capacity_increase); + let align = mem::align_of::(); + let new_size = new_cap * elem_size; + let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, align) }; + let (_, new_cap) = self.a.usable_size(&new_layout);; + alloc_guard(new_cap); + match self.a.alloc_array::(new_cap) { + Ok(ptr) => (new_cap, ptr), + Err(e) => self.a.oom(e), + } + } + }; + self.ptr = uniq; + self.cap = new_cap; + } + + /// Increases the size of the type's backing allocation. This is common enough /// to want to do that it's easiest to just have a dedicated method. Slightly /// more efficient logic can be provided for this than the general case. /// @@ -286,53 +339,77 @@ impl RawVec { #[inline(never)] #[cold] pub fn double(&mut self) { - unsafe { - let elem_size = mem::size_of::(); + let cap = self.cap; + self.grow_by(cap) + } - // since we set the capacity to usize::MAX when elem_size is - // 0, getting to here necessarily means the RawVec is overfull. - assert!(elem_size != 0, "capacity overflow"); - - let (new_cap, uniq) = match self.current_layout() { - Some(cur) => { - // Since we guarantee that we never allocate more than - // isize::MAX bytes, `elem_size * self.cap <= isize::MAX` as - // a precondition, so this can't overflow. Additionally the - // alignment will never be too large as to "not be - // satisfiable", so `Layout::from_size_align` will always - // return `Some`. - // - // tl;dr; we bypass runtime checks due to dynamic assertions - // in this module, allowing us to use - // `from_size_align_unchecked`. - let new_cap = 2 * self.cap; - let new_size = new_cap * elem_size; - let new_layout = Layout::from_size_align_unchecked(new_size, cur.align()); - alloc_guard(new_size); - let ptr_res = self.a.realloc(self.ptr.as_ptr() as *mut u8, - cur, - new_layout); - match ptr_res { - Ok(ptr) => (new_cap, Unique::new_unchecked(ptr as *mut T)), - Err(e) => self.a.oom(e), - } - } - None => { - // skip to 4 because tiny Vec's are dumb; but not if that - // would cause overflow - let new_cap = if elem_size > (!0) / 8 { 1 } else { 4 }; - match self.a.alloc_array::(new_cap) { - Ok(ptr) => (new_cap, ptr), - Err(e) => self.a.oom(e), - } - } - }; - self.ptr = uniq; - self.cap = new_cap; - } + /// Given a `current_capacity` and a desired `capacity_increase` returns a + /// suitable capacity for the `RawVec` such that `suitable_capacity >= + /// current_capacity + capacity_increase`. + /// + /// # Panics + /// + /// Panics on overflow if `current_capacity + capacity_increase > + /// std::usize::MAX`. + /// + /// + /// # Growth strategy + /// + /// RawVec grows differently depending on: + /// + /// - 1. initial size: grows from zero to at least 64 bytes; + /// use `with_capacity` to avoid a growth from zero. + /// + /// - 2. vector size: + /// - small vectors (<= 4096 bytes) and large vectors (>= 4096 * 32 bytes) + /// grow with a growth factor of 2x. + /// - otherwise (medium-sized vectors) grow with a growth factor of 1.5x. + /// + /// # Growth factor + /// + /// Medium-sized vectors' growth-factor is chosen to allow reusing memory from + /// previous allocations. Previously freed memory can be reused after + /// + /// - 4 reallocations for a growth factor of 1.5x + /// - 3 reallocations for a growth factor of 1.45x + /// - 2 reallocations for a growth factor of 1.3x + /// + /// Which one is better [is application + /// dependent](https://stackoverflow.com/questions/1100311/what-is-the-ideal-growth-rate-for-a-dynamically-allocated-array), + /// also some claim that [the golden ration (1.618) is + /// optimal](https://crntaylor.wordpress.com/2011/07/15/optimal-memory-reallocation-and-the-golden-ratio/). + /// The trade-off is having to wait for many reallocations to be able to + /// reuse old memory. + /// + /// Note: a factor of 2x _never_ allows reusing previously-freed memory. + /// + #[inline] + fn suitable_capacity(current_capacity: usize, capacity_increase: usize) -> usize { + let elem_size = mem::size_of::(); + assert!(elem_size != 0, "RawVecs of zero-sized types can't grow"); + + // Computes the capacity from the `current_capacity` following the + // growth-strategy: + let growth_capacity = match current_capacity { + // Empty vector => at least 64 bytes + 0 => (64 / elem_size).max(1), + // Small and large vectors (<= 4096 bytes, and >= 4096 * 32 bytes): + // + // FIXME: jemalloc specific behavior, allocators should provide a + // way to query the byte size of blocks that can grow inplace. + // + // jemalloc can never grow in place small blocks but blocks larger + // than or equal to 4096 bytes can be expanded in place: + c if c < 4096 / elem_size => 2 * c, + c if c > 4096 * 32 / elem_size => 2 * c, + // Medium sized vectors in the [4096, 4096 * 32) bytes range: + c => (c * 3 + 1) / 2 + }; + + growth_capacity.max(current_capacity + capacity_increase) } - /// Attempts to double the size of the type's backing allocation in place. This is common + /// Attempts to increase the size of the type's backing allocation in place. This is common /// enough to want to do that it's easiest to just have a dedicated method. Slightly /// more efficient logic can be provided for this than the general case. /// @@ -344,41 +421,50 @@ impl RawVec { /// all `usize::MAX` slots in your imaginary buffer. /// * Panics on 32-bit platforms if the requested capacity exceeds /// `isize::MAX` bytes. - #[inline(never)] - #[cold] pub fn double_in_place(&mut self) -> bool { - unsafe { - let elem_size = mem::size_of::(); - let old_layout = match self.current_layout() { - Some(layout) => layout, - None => return false, // nothing to double - }; - - // since we set the capacity to usize::MAX when elem_size is - // 0, getting to here necessarily means the RawVec is overfull. - assert!(elem_size != 0, "capacity overflow"); + let capacity_increase = self.cap; + self.grow_by_in_place(capacity_increase) + } - // Since we guarantee that we never allocate more than isize::MAX - // bytes, `elem_size * self.cap <= isize::MAX` as a precondition, so - // this can't overflow. - // - // Similarly like with `double` above we can go straight to - // `Layout::from_size_align_unchecked` as we know this won't - // overflow and the alignment is sufficiently small. - let new_cap = 2 * self.cap; - let new_size = new_cap * elem_size; - alloc_guard(new_size); - let ptr = self.ptr() as *mut _; - let new_layout = Layout::from_size_align_unchecked(new_size, old_layout.align()); - match self.a.grow_in_place(ptr, old_layout, new_layout) { - Ok(_) => { - // We can't directly divide `size`. - self.cap = new_cap; - true - } - Err(_) => { - false - } + /// Attempts to grow the vector capacity by `capacity_increase`. + /// + /// It allows implementing amortized O(1) `push` on vector-like containers. + /// + /// # Attributes + /// + /// - `#[inline]`: LLVM is able to completely elide memory allocations in + /// many cases if it can "see" where memory is allocated and freed. + /// - `#[cold]`: calling this function is a "rare" event + /// + #[inline(never)] + #[cold] + pub fn grow_by_in_place(&mut self, capacity_increase: usize) -> bool { + let elem_size = mem::size_of::(); + assert!(elem_size != 0, "RawVecs of zero-sized types can't grow"); + let old_layout = match self.current_layout() { + Some(layout) => layout, + None => return false, // nothing to grow + }; + + // The invariant `elem_size * self.cap <= isize::MAX` is + // maintained by `alloc_guard`; the alignment will never be too + // large as to "not be satisfiable" (so we can use + // `from_size_align_unchecked`). + let new_cap = Self::suitable_capacity(self.cap, capacity_increase); + let new_size = new_cap * elem_size; + let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, old_layout.align()) }; + let (_, usable_size) = self.a.usable_size(&new_layout); + let new_layout = unsafe { Layout::from_size_align_unchecked(usable_size, old_layout.align()) }; + alloc_guard(usable_size); + let ptr = self.ptr() as *mut _; + match unsafe { self.a.grow_in_place(ptr, old_layout, new_layout) } { + Ok(_) => { + // We can't directly divide `size`. + self.cap = new_cap; + true + } + Err(_) => { + false } } } @@ -794,4 +880,5 @@ mod tests { } + } diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 5aca199cf40c0..d442964091dc3 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -79,6 +79,7 @@ use core::ops; use core::ptr; use core::ptr::Shared; use core::slice; +use core::intrinsics; use borrow::ToOwned; use borrow::Cow; @@ -707,6 +708,11 @@ impl Vec { self.pop().unwrap() } + #[inline(always)] + fn is_full(&self) -> bool { + unsafe { intrinsics::unlikely(self.len == self.buf.cap()) } + } + /// Inserts an element at position `index` within the vector, shifting all /// elements after it to the right. /// @@ -729,8 +735,8 @@ impl Vec { assert!(index <= len); // space for the new element - if len == self.buf.cap() { - self.buf.double(); + if self.is_full() { + self.buf.grow_by(1); } unsafe { @@ -964,8 +970,8 @@ impl Vec { pub fn push(&mut self, value: T) { // This will panic or abort if we would allocate > isize::MAX bytes // or if the length increment would overflow for zero-sized types. - if self.len == self.buf.cap() { - self.buf.double(); + if self.is_full() { + self.buf.grow_by(1); } unsafe { let end = self.as_mut_ptr().offset(self.len as isize); @@ -2532,8 +2538,8 @@ impl<'a, T> Placer for PlaceBack<'a, T> { fn make_place(self) -> Self { // This will panic or abort if we would allocate > isize::MAX bytes // or if the length increment would overflow for zero-sized types. - if self.vec.len == self.vec.buf.cap() { - self.vec.buf.double(); + if self.vec.is_full() { + self.vec.buf.grow_by(1); } self } diff --git a/src/liballoc/vec_deque.rs b/src/liballoc/vec_deque.rs index f56aa23a4eb2f..3defa1a751165 100644 --- a/src/liballoc/vec_deque.rs +++ b/src/liballoc/vec_deque.rs @@ -25,6 +25,7 @@ use core::ops::{Index, IndexMut, Place, Placer, InPlace}; use core::ptr; use core::ptr::Shared; use core::slice; +use core::intrinsics; use core::hash::{Hash, Hasher}; use core::cmp; @@ -139,7 +140,7 @@ impl VecDeque { /// Returns `true` if and only if the buffer is at full capacity. #[inline] fn is_full(&self) -> bool { - self.cap() - self.len() == 1 + unsafe { intrinsics::unlikely(self.cap() - self.len() == 1) } } /// Returns the index in the underlying buffer for a given logical element @@ -1754,7 +1755,7 @@ impl VecDeque { fn grow_if_necessary(&mut self) { if self.is_full() { let old_cap = self.cap(); - self.buf.double(); + self.buf.grow_by(1); unsafe { self.handle_cap_increase(old_cap); } From 0a9d79279c8689cc3ec0e8345a017f8b3b6d377d Mon Sep 17 00:00:00 2001 From: gnzlbg Date: Sat, 21 Oct 2017 23:21:40 +0200 Subject: [PATCH 02/12] style --- src/liballoc/raw_vec.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/liballoc/raw_vec.rs b/src/liballoc/raw_vec.rs index 8d02e916ddf5b..546a6f2ebdcec 100644 --- a/src/liballoc/raw_vec.rs +++ b/src/liballoc/raw_vec.rs @@ -259,15 +259,21 @@ impl RawVec { // `from_size_align_unchecked`). let new_cap = Self::suitable_capacity(self.cap, capacity_increase); let new_size = new_cap * elem_size; - let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, cur.align()) }; + let new_layout = unsafe { + Layout::from_size_align_unchecked(new_size, cur.align()) + }; let (_, usable_size) = self.a.usable_size(&new_layout); - let new_layout = unsafe { Layout::from_size_align_unchecked(usable_size, cur.align()) }; + let new_layout = unsafe { + Layout::from_size_align_unchecked(usable_size, cur.align()) + }; alloc_guard(usable_size); let ptr_res = unsafe { self.a.realloc(self.ptr.as_ptr() as *mut u8, cur, new_layout) }; match ptr_res { - Ok(ptr) => (new_cap, unsafe { Unique::new_unchecked(ptr as *mut T) }), + Ok(ptr) => (new_cap, unsafe { + Unique::new_unchecked(ptr as *mut T) + }), Err(e) => self.a.oom(e), } } @@ -351,7 +357,7 @@ impl RawVec { /// /// Panics on overflow if `current_capacity + capacity_increase > /// std::usize::MAX`. - /// + /// /// /// # Growth strategy /// @@ -452,9 +458,13 @@ impl RawVec { // `from_size_align_unchecked`). let new_cap = Self::suitable_capacity(self.cap, capacity_increase); let new_size = new_cap * elem_size; - let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, old_layout.align()) }; + let new_layout = unsafe { + Layout::from_size_align_unchecked(new_size, old_layout.align()) + }; let (_, usable_size) = self.a.usable_size(&new_layout); - let new_layout = unsafe { Layout::from_size_align_unchecked(usable_size, old_layout.align()) }; + let new_layout = unsafe { + Layout::from_size_align_unchecked(usable_size, old_layout.align()) + }; alloc_guard(usable_size); let ptr = self.ptr() as *mut _; match unsafe { self.a.grow_in_place(ptr, old_layout, new_layout) } { From 5a0502cd9f061cf9e5905680d7c0921f6a63cbf8 Mon Sep 17 00:00:00 2001 From: gnzlbg Date: Sat, 21 Oct 2017 23:28:35 +0200 Subject: [PATCH 03/12] break doc links --- src/liballoc/raw_vec.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/liballoc/raw_vec.rs b/src/liballoc/raw_vec.rs index 546a6f2ebdcec..b1441ebf4be0b 100644 --- a/src/liballoc/raw_vec.rs +++ b/src/liballoc/raw_vec.rs @@ -381,9 +381,11 @@ impl RawVec { /// - 2 reallocations for a growth factor of 1.3x /// /// Which one is better [is application - /// dependent](https://stackoverflow.com/questions/1100311/what-is-the-ideal-growth-rate-for-a-dynamically-allocated-array), + /// dependent](https://stackoverflow.com/questions/1100311/ + /// what-is-the-ideal-growth-rate-for-a-dynamically-allocated-array), /// also some claim that [the golden ration (1.618) is - /// optimal](https://crntaylor.wordpress.com/2011/07/15/optimal-memory-reallocation-and-the-golden-ratio/). + /// optimal](https://crntaylor.wordpress.com/2011/07/15/ + /// optimal-memory-reallocation-and-the-golden-ratio/). /// The trade-off is having to wait for many reallocations to be able to /// reuse old memory. /// From a52f06efb6d2245401f49b40c3a57c9bd95d7523 Mon Sep 17 00:00:00 2001 From: gnzlbg Date: Sun, 22 Oct 2017 09:04:40 +0200 Subject: [PATCH 04/12] vecdeque grow_by(old_cap) --- src/liballoc/vec_deque.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/liballoc/vec_deque.rs b/src/liballoc/vec_deque.rs index 3defa1a751165..59fddd3ee6c37 100644 --- a/src/liballoc/vec_deque.rs +++ b/src/liballoc/vec_deque.rs @@ -1755,7 +1755,7 @@ impl VecDeque { fn grow_if_necessary(&mut self) { if self.is_full() { let old_cap = self.cap(); - self.buf.grow_by(1); + self.buf.grow_by(old_cap); unsafe { self.handle_cap_increase(old_cap); } From ebab7b3cf977a83ebe87a2ad31c70f700b26014c Mon Sep 17 00:00:00 2001 From: gnzlbg Date: Sun, 22 Oct 2017 09:25:27 +0200 Subject: [PATCH 05/12] move unlikely intrinsics out of is_full --- src/liballoc/vec.rs | 8 ++++---- src/liballoc/vec_deque.rs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index d442964091dc3..471e382454d07 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -710,7 +710,7 @@ impl Vec { #[inline(always)] fn is_full(&self) -> bool { - unsafe { intrinsics::unlikely(self.len == self.buf.cap()) } + self.len == self.buf.cap() } /// Inserts an element at position `index` within the vector, shifting all @@ -735,7 +735,7 @@ impl Vec { assert!(index <= len); // space for the new element - if self.is_full() { + if unsafe { intrinsics::unlikely(self.is_full()) } { self.buf.grow_by(1); } @@ -970,7 +970,7 @@ impl Vec { pub fn push(&mut self, value: T) { // This will panic or abort if we would allocate > isize::MAX bytes // or if the length increment would overflow for zero-sized types. - if self.is_full() { + if unsafe { intrinsics::unlikely(self.is_full()) } { self.buf.grow_by(1); } unsafe { @@ -2538,7 +2538,7 @@ impl<'a, T> Placer for PlaceBack<'a, T> { fn make_place(self) -> Self { // This will panic or abort if we would allocate > isize::MAX bytes // or if the length increment would overflow for zero-sized types. - if self.vec.is_full() { + if unsafe { intrinsics::unlikely(self.vec.is_full()) } { self.vec.buf.grow_by(1); } self diff --git a/src/liballoc/vec_deque.rs b/src/liballoc/vec_deque.rs index 59fddd3ee6c37..169c943940937 100644 --- a/src/liballoc/vec_deque.rs +++ b/src/liballoc/vec_deque.rs @@ -140,7 +140,7 @@ impl VecDeque { /// Returns `true` if and only if the buffer is at full capacity. #[inline] fn is_full(&self) -> bool { - unsafe { intrinsics::unlikely(self.cap() - self.len() == 1) } + self.cap() - self.len() == 1 } /// Returns the index in the underlying buffer for a given logical element @@ -1753,7 +1753,7 @@ impl VecDeque { // This may panic or abort #[inline] fn grow_if_necessary(&mut self) { - if self.is_full() { + if unsafe { intrinsics::unlikely(self.is_full()) } { let old_cap = self.cap(); self.buf.grow_by(old_cap); unsafe { From 30aae7bbec88fc4383fdfe3b752f5ecf89e645a8 Mon Sep 17 00:00:00 2001 From: gnzlbg Date: Sun, 22 Oct 2017 20:41:22 +0200 Subject: [PATCH 06/12] improve reserve instead --- src/liballoc/raw_vec.rs | 451 ++++++++++++++++++++------------------ src/liballoc/vec.rs | 7 +- src/liballoc/vec_deque.rs | 2 +- 3 files changed, 240 insertions(+), 220 deletions(-) diff --git a/src/liballoc/raw_vec.rs b/src/liballoc/raw_vec.rs index b1441ebf4be0b..9e9eda06121ed 100644 --- a/src/liballoc/raw_vec.rs +++ b/src/liballoc/raw_vec.rs @@ -165,6 +165,78 @@ impl RawVec { } } +/// Given a `current_capacity` and a desired `capacity_increase` returns a +/// suitable capacity for the `RawVec` such that `suitable_capacity >= +/// current_capacity + capacity_increase`. +/// +/// # Panics +/// +/// Panics on overflow if `current_capacity + capacity_increase > +/// std::usize::MAX`. +/// +/// +/// # Growth strategy +/// +/// RawVec grows differently depending on: +/// +/// - 1. initial size: grows from zero to at least 64 bytes; +/// use `with_capacity` to avoid a growth from zero. +/// +/// - 2. vector size: +/// - small vectors (<= 4096 bytes) and large vectors (>= 4096 * 32 bytes) +/// grow with a growth factor of 2x. +/// - otherwise (medium-sized vectors) grow with a growth factor of 1.5x. +/// +/// # Growth factor +/// +/// Medium-sized vectors' growth-factor is chosen to allow reusing memory from +/// previous allocations. Previously freed memory can be reused after +/// +/// - 4 reallocations for a growth factor of 1.5x +/// - 3 reallocations for a growth factor of 1.45x +/// - 2 reallocations for a growth factor of 1.3x +/// +/// Which one is better [is application +/// dependent](https://stackoverflow.com/questions/1100311/ +/// what-is-the-ideal-growth-rate-for-a-dynamically-allocated-array), +/// also some claim that [the golden ration (1.618) is +/// optimal](https://crntaylor.wordpress.com/2011/07/15/ +/// optimal-memory-reallocation-and-the-golden-ratio/). +/// The trade-off is having to wait for many reallocations to be able to +/// reuse old memory. +/// +/// Note: a factor of 2x _never_ allows reusing previously-freed memory. +/// +#[inline(always)] +fn amortized_new_capacity(elem_size: usize, current_capacity: usize, + capacity_increase: usize) -> usize { + assert!(elem_size != 0, "RawVecs of zero-sized types can't grow"); + + // Computes the capacity from the `current_capacity` following the + // growth-strategy. The function `alloc_guard` ensures that + // `current_capacity <= isize::MAX` so that `current_capacity * N` where `N + // <= 2` cannot overflow. + let growth_capacity = match current_capacity { + // Empty vector => at least 64 bytes + // [OLD]: 0 => if elem_size > (!0) / 8 { 1 } else { 4 }, + 0 => (64 / elem_size).max(1), + // Small and large vectors (<= 4096 bytes, and >= 4096 * 32 bytes): + // + // FIXME: jemalloc specific behavior, allocators should provide a + // way to query the byte size of blocks that can grow inplace. + // + // jemalloc can never grow in place small blocks but blocks larger + // than or equal to 4096 bytes can be expanded in place: + c if c < 4096 / elem_size => 2 * c, + c if c > 4096 * 32 / elem_size => 2 * c, + // Medium sized vectors in the [4096, 4096 * 32) bytes range: + c => c / 2 * 3 + }; + cmp::max(growth_capacity, + current_capacity.checked_add(capacity_increase).unwrap()) +} + + impl RawVec { /// Reconstitutes a RawVec from a pointer, capacity. /// @@ -235,66 +307,7 @@ impl RawVec { } } - /// Grows the vector capacity by `capacity_increase`. - /// - /// It allows implementing amortized O(1) `push` on vector-like containers. - /// - /// # Attributes - /// - /// - `#[inline]`: LLVM is able to completely elide memory allocations in - /// many cases if it can "see" where memory is allocated and freed. - /// - `#[cold]`: calling this function is a "rare" event - /// - #[inline] - #[cold] - pub fn grow_by(&mut self, capacity_increase: usize) { - let elem_size = mem::size_of::(); - assert!(elem_size != 0, "RawVecs of zero-sized types can't grow"); - - let (new_cap, uniq) = match self.current_layout() { - Some(cur) => { - // The invariant `elem_size * self.cap <= isize::MAX` is - // maintained by `alloc_guard`; the alignment will never be too - // large as to "not be specifiable" (so we can use - // `from_size_align_unchecked`). - let new_cap = Self::suitable_capacity(self.cap, capacity_increase); - let new_size = new_cap * elem_size; - let new_layout = unsafe { - Layout::from_size_align_unchecked(new_size, cur.align()) - }; - let (_, usable_size) = self.a.usable_size(&new_layout); - let new_layout = unsafe { - Layout::from_size_align_unchecked(usable_size, cur.align()) - }; - alloc_guard(usable_size); - let ptr_res = unsafe { self.a.realloc(self.ptr.as_ptr() as *mut u8, - cur, - new_layout) }; - match ptr_res { - Ok(ptr) => (new_cap, unsafe { - Unique::new_unchecked(ptr as *mut T) - }), - Err(e) => self.a.oom(e), - } - } - None => { - let new_cap = Self::suitable_capacity(self.cap, capacity_increase); - let align = mem::align_of::(); - let new_size = new_cap * elem_size; - let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, align) }; - let (_, new_cap) = self.a.usable_size(&new_layout);; - alloc_guard(new_cap); - match self.a.alloc_array::(new_cap) { - Ok(ptr) => (new_cap, ptr), - Err(e) => self.a.oom(e), - } - } - }; - self.ptr = uniq; - self.cap = new_cap; - } - - /// Increases the size of the type's backing allocation. This is common enough + /// Doubles the size of the type's backing allocation. This is common enough /// to want to do that it's easiest to just have a dedicated method. Slightly /// more efficient logic can be provided for this than the general case. /// @@ -345,79 +358,53 @@ impl RawVec { #[inline(never)] #[cold] pub fn double(&mut self) { - let cap = self.cap; - self.grow_by(cap) - } + unsafe { + let elem_size = mem::size_of::(); - /// Given a `current_capacity` and a desired `capacity_increase` returns a - /// suitable capacity for the `RawVec` such that `suitable_capacity >= - /// current_capacity + capacity_increase`. - /// - /// # Panics - /// - /// Panics on overflow if `current_capacity + capacity_increase > - /// std::usize::MAX`. - /// - /// - /// # Growth strategy - /// - /// RawVec grows differently depending on: - /// - /// - 1. initial size: grows from zero to at least 64 bytes; - /// use `with_capacity` to avoid a growth from zero. - /// - /// - 2. vector size: - /// - small vectors (<= 4096 bytes) and large vectors (>= 4096 * 32 bytes) - /// grow with a growth factor of 2x. - /// - otherwise (medium-sized vectors) grow with a growth factor of 1.5x. - /// - /// # Growth factor - /// - /// Medium-sized vectors' growth-factor is chosen to allow reusing memory from - /// previous allocations. Previously freed memory can be reused after - /// - /// - 4 reallocations for a growth factor of 1.5x - /// - 3 reallocations for a growth factor of 1.45x - /// - 2 reallocations for a growth factor of 1.3x - /// - /// Which one is better [is application - /// dependent](https://stackoverflow.com/questions/1100311/ - /// what-is-the-ideal-growth-rate-for-a-dynamically-allocated-array), - /// also some claim that [the golden ration (1.618) is - /// optimal](https://crntaylor.wordpress.com/2011/07/15/ - /// optimal-memory-reallocation-and-the-golden-ratio/). - /// The trade-off is having to wait for many reallocations to be able to - /// reuse old memory. - /// - /// Note: a factor of 2x _never_ allows reusing previously-freed memory. - /// - #[inline] - fn suitable_capacity(current_capacity: usize, capacity_increase: usize) -> usize { - let elem_size = mem::size_of::(); - assert!(elem_size != 0, "RawVecs of zero-sized types can't grow"); - - // Computes the capacity from the `current_capacity` following the - // growth-strategy: - let growth_capacity = match current_capacity { - // Empty vector => at least 64 bytes - 0 => (64 / elem_size).max(1), - // Small and large vectors (<= 4096 bytes, and >= 4096 * 32 bytes): - // - // FIXME: jemalloc specific behavior, allocators should provide a - // way to query the byte size of blocks that can grow inplace. - // - // jemalloc can never grow in place small blocks but blocks larger - // than or equal to 4096 bytes can be expanded in place: - c if c < 4096 / elem_size => 2 * c, - c if c > 4096 * 32 / elem_size => 2 * c, - // Medium sized vectors in the [4096, 4096 * 32) bytes range: - c => (c * 3 + 1) / 2 - }; - - growth_capacity.max(current_capacity + capacity_increase) + // since we set the capacity to usize::MAX when elem_size is + // 0, getting to here necessarily means the RawVec is overfull. + assert!(elem_size != 0, "capacity overflow"); + + let (new_cap, uniq) = match self.current_layout() { + Some(cur) => { + // Since we guarantee that we never allocate more than + // isize::MAX bytes, `elem_size * self.cap <= isize::MAX` as + // a precondition, so this can't overflow. Additionally the + // alignment will never be too large as to "not be + // satisfiable", so `Layout::from_size_align` will always + // return `Some`. + // + // tl;dr; we bypass runtime checks due to dynamic assertions + // in this module, allowing us to use + // `from_size_align_unchecked`. + let new_cap = 2 * self.cap; + let new_size = new_cap * elem_size; + let new_layout = Layout::from_size_align_unchecked(new_size, cur.align()); + alloc_guard(new_size); + let ptr_res = self.a.realloc(self.ptr.as_ptr() as *mut u8, + cur, + new_layout); + match ptr_res { + Ok(ptr) => (new_cap, Unique::new_unchecked(ptr as *mut T)), + Err(e) => self.a.oom(e), + } + } + None => { + // skip to 4 because tiny Vec's are dumb; but not if that + // would cause overflow + let new_cap = if elem_size > (!0) / 8 { 1 } else { 4 }; + match self.a.alloc_array::(new_cap) { + Ok(ptr) => (new_cap, ptr), + Err(e) => self.a.oom(e), + } + } + }; + self.ptr = uniq; + self.cap = new_cap; + } } - /// Attempts to increase the size of the type's backing allocation in place. This is common + /// Attempts to double the size of the type's backing allocation in place. This is common /// enough to want to do that it's easiest to just have a dedicated method. Slightly /// more efficient logic can be provided for this than the general case. /// @@ -429,54 +416,41 @@ impl RawVec { /// all `usize::MAX` slots in your imaginary buffer. /// * Panics on 32-bit platforms if the requested capacity exceeds /// `isize::MAX` bytes. - pub fn double_in_place(&mut self) -> bool { - let capacity_increase = self.cap; - self.grow_by_in_place(capacity_increase) - } - - /// Attempts to grow the vector capacity by `capacity_increase`. - /// - /// It allows implementing amortized O(1) `push` on vector-like containers. - /// - /// # Attributes - /// - /// - `#[inline]`: LLVM is able to completely elide memory allocations in - /// many cases if it can "see" where memory is allocated and freed. - /// - `#[cold]`: calling this function is a "rare" event - /// #[inline(never)] #[cold] - pub fn grow_by_in_place(&mut self, capacity_increase: usize) -> bool { - let elem_size = mem::size_of::(); - assert!(elem_size != 0, "RawVecs of zero-sized types can't grow"); - let old_layout = match self.current_layout() { - Some(layout) => layout, - None => return false, // nothing to grow - }; - - // The invariant `elem_size * self.cap <= isize::MAX` is - // maintained by `alloc_guard`; the alignment will never be too - // large as to "not be satisfiable" (so we can use - // `from_size_align_unchecked`). - let new_cap = Self::suitable_capacity(self.cap, capacity_increase); - let new_size = new_cap * elem_size; - let new_layout = unsafe { - Layout::from_size_align_unchecked(new_size, old_layout.align()) - }; - let (_, usable_size) = self.a.usable_size(&new_layout); - let new_layout = unsafe { - Layout::from_size_align_unchecked(usable_size, old_layout.align()) - }; - alloc_guard(usable_size); - let ptr = self.ptr() as *mut _; - match unsafe { self.a.grow_in_place(ptr, old_layout, new_layout) } { - Ok(_) => { - // We can't directly divide `size`. - self.cap = new_cap; - true - } - Err(_) => { - false + pub fn double_in_place(&mut self) -> bool { + unsafe { + let elem_size = mem::size_of::(); + let old_layout = match self.current_layout() { + Some(layout) => layout, + None => return false, // nothing to double + }; + + // since we set the capacity to usize::MAX when elem_size is + // 0, getting to here necessarily means the RawVec is overfull. + assert!(elem_size != 0, "capacity overflow"); + + // Since we guarantee that we never allocate more than isize::MAX + // bytes, `elem_size * self.cap <= isize::MAX` as a precondition, so + // this can't overflow. + // + // Similarly like with `double` above we can go straight to + // `Layout::from_size_align_unchecked` as we know this won't + // overflow and the alignment is sufficiently small. + let new_cap = 2 * self.cap; + let new_size = new_cap * elem_size; + alloc_guard(new_size); + let ptr = self.ptr() as *mut _; + let new_layout = Layout::from_size_align_unchecked(new_size, old_layout.align()); + match self.a.grow_in_place(ptr, old_layout, new_layout) { + Ok(_) => { + // We can't directly divide `size`. + self.cap = new_cap; + true + } + Err(_) => { + false + } } } } @@ -537,19 +511,6 @@ impl RawVec { } } - /// Calculates the buffer's new size given that it'll hold `used_cap + - /// needed_extra_cap` elements. This logic is used in amortized reserve methods. - /// Returns `(new_capacity, new_alloc_size)`. - fn amortized_new_size(&self, used_cap: usize, needed_extra_cap: usize) -> usize { - // Nothing we can really do about these checks :( - let required_cap = used_cap.checked_add(needed_extra_cap) - .expect("capacity overflow"); - // Cannot overflow, because `cap <= isize::MAX`, and type of `cap` is `usize`. - let double_cap = self.cap * 2; - // `double_cap` guarantees exponential growth. - cmp::max(double_cap, required_cap) - } - /// Ensures that the buffer contains at least enough space to hold /// `used_cap + needed_extra_cap` elements. If it doesn't already have /// enough capacity, will reallocate enough space plus comfortable slack @@ -602,6 +563,8 @@ impl RawVec { /// # vector.push_all(&[1, 3, 5, 7, 9]); /// # } /// ``` + #[inline] + #[cold] pub fn reserve(&mut self, used_cap: usize, needed_extra_cap: usize) { unsafe { // NOTE: we don't early branch on ZSTs here because we want this @@ -615,8 +578,17 @@ impl RawVec { return; } - let new_cap = self.amortized_new_size(used_cap, needed_extra_cap); + let elem_size = mem::size_of::(); + let new_cap = amortized_new_capacity(elem_size, + used_cap, + needed_extra_cap); + let new_layout = match Layout::array::(new_cap) { + Some(layout) => layout, + None => panic!("capacity overflow"), + }; + let (_, usable_size) = self.a.usable_size(&new_layout); + let new_cap = usable_size / elem_size; let new_layout = match Layout::array::(new_cap) { Some(layout) => layout, None => panic!("capacity overflow"), @@ -674,7 +646,10 @@ impl RawVec { return false; } - let new_cap = self.amortized_new_size(used_cap, needed_extra_cap); + let elem_size = mem::size_of::(); + let new_cap = amortized_new_capacity(elem_size, + used_cap, + needed_extra_cap); // Here, `cap < used_cap + needed_extra_cap <= new_cap` // (regardless of whether `self.cap - used_cap` wrapped). @@ -682,6 +657,12 @@ impl RawVec { let ptr = self.ptr() as *mut _; let new_layout = Layout::new::().repeat(new_cap).unwrap().0; + let (_, usable_size) = self.a.usable_size(&new_layout); + let new_cap = usable_size / elem_size; + let new_layout = match Layout::array::(new_cap) { + Some(layout) => layout, + None => panic!("capacity overflow"), + }; // FIXME: may crash and burn on over-reserve alloc_guard(new_layout.size()); match self.a.grow_in_place(ptr, old_layout, new_layout) { @@ -860,37 +841,75 @@ mod tests { } #[test] - fn reserve_does_not_overallocate() { + fn amortized_new_capacity_tests() { + // empty vector: + assert_eq!(amortized_new_capacity(1, 0, 1), 64); + assert_eq!(amortized_new_capacity(64, 0, 1), 1); + assert_eq!(amortized_new_capacity(128, 0, 1), 1); + + // small vector: + assert_eq!(amortized_new_capacity(1, 2048, 1), 4096); // 2x + + // medium vector + assert_eq!(amortized_new_capacity(1, 5000, 1), 7500); // 1.5x + + // large vector + assert_eq!(amortized_new_capacity(1, 140000, 1), 280000); // 2x + } + + #[test] + fn reserve_() { + // Allocation from empty vector: { - let mut v: RawVec = RawVec::new(); - // First `reserve` allocates like `reserve_exact` - v.reserve(0, 9); - assert_eq!(9, v.cap()); - } + let mut v: RawVec = RawVec::new(); + assert_eq!(v.cap(), 0); + // empty vectors allocate at least 64 bytes + v.reserve(0, 63); + assert_eq!(v.cap(), 64); + } { let mut v: RawVec = RawVec::new(); - v.reserve(0, 7); - assert_eq!(7, v.cap()); - // 97 if more than double of 7, so `reserve` should work - // like `reserve_exact`. - v.reserve(7, 90); - assert_eq!(97, v.cap()); - } + // empty vectors allocate at least 64 bytes + v.reserve(0, 9); + assert_eq!(16, v.cap()); + } + // Exponential growth: { - let mut v: RawVec = RawVec::new(); - v.reserve(0, 12); - assert_eq!(12, v.cap()); - v.reserve(12, 3); - // 3 is less than half of 12, so `reserve` must grow - // exponentially. At the time of writing this test grow - // factor is 2, so new capacity is 24, however, grow factor - // of 1.5 is OK too. Hence `>= 18` in assert. - assert!(v.cap() >= 12 + 12 / 2); + let mut v: RawVec = RawVec::new(); + assert_eq!(v.cap(), 0); + + // empty vectors allocate at least 64 bytes + v.reserve(0, 1); + assert_eq!(v.cap(), 64); + + // small vectors grow with a factor of two: + v.reserve(64, 1); + assert_eq!(v.cap(), 128); + + // The required capacity is 128 + 3968 = 4096 which is larger than + // 2*128 = 256, so here `reserve` allocates "exactly" 4096 elements + // modulo extra size returned by the allocator: + v.reserve(128, 3968); + assert!(v.cap() >= 4096 && v.cap() <= 4096 + 128); + + // 1 <= 1.5 * cap, so here the "medium" sized vector uses a growth + // factor of 1.5: + let cap = v.cap(); + v.reserve(cap, 1); + assert!(v.cap() >= cap * 3 / 2 && v.cap() <= cap * 3 / 2 + 512); + + // we reserve enough to make the vector "large" + let cap = v.cap(); + v.reserve(cap, 4096 * 32); + assert!(v.cap() >= cap + 4096 * 32 + && v.cap() <= cap + 4096 * 34); + + // large vectors grow with a growth-factor equals 2x: + let cap = v.cap(); + v.reserve(cap, 1); + assert!(v.cap() >= cap * 2 && v.cap() <= cap * 2 + 4096); } } - - - } diff --git a/src/liballoc/vec.rs b/src/liballoc/vec.rs index 471e382454d07..7b8b39ba29ae7 100644 --- a/src/liballoc/vec.rs +++ b/src/liballoc/vec.rs @@ -736,7 +736,7 @@ impl Vec { // space for the new element if unsafe { intrinsics::unlikely(self.is_full()) } { - self.buf.grow_by(1); + self.buf.reserve(self.len, 1); } unsafe { @@ -971,7 +971,7 @@ impl Vec { // This will panic or abort if we would allocate > isize::MAX bytes // or if the length increment would overflow for zero-sized types. if unsafe { intrinsics::unlikely(self.is_full()) } { - self.buf.grow_by(1); + self.buf.reserve(self.len, 1); } unsafe { let end = self.as_mut_ptr().offset(self.len as isize); @@ -2539,7 +2539,8 @@ impl<'a, T> Placer for PlaceBack<'a, T> { // This will panic or abort if we would allocate > isize::MAX bytes // or if the length increment would overflow for zero-sized types. if unsafe { intrinsics::unlikely(self.vec.is_full()) } { - self.vec.buf.grow_by(1); + let len = self.vec.len(); + self.vec.buf.reserve(len, 1); } self } diff --git a/src/liballoc/vec_deque.rs b/src/liballoc/vec_deque.rs index 169c943940937..c4848fc1097bf 100644 --- a/src/liballoc/vec_deque.rs +++ b/src/liballoc/vec_deque.rs @@ -1755,7 +1755,7 @@ impl VecDeque { fn grow_if_necessary(&mut self) { if unsafe { intrinsics::unlikely(self.is_full()) } { let old_cap = self.cap(); - self.buf.grow_by(old_cap); + self.buf.double(); unsafe { self.handle_cap_increase(old_cap); } From 9da3665e703241e0cd9a707038232b104a115c43 Mon Sep 17 00:00:00 2001 From: gnzlbg Date: Sun, 22 Oct 2017 23:21:09 +0200 Subject: [PATCH 07/12] all tests pass --- src/liballoc/raw_vec.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/liballoc/raw_vec.rs b/src/liballoc/raw_vec.rs index 9e9eda06121ed..603bea21e9aa7 100644 --- a/src/liballoc/raw_vec.rs +++ b/src/liballoc/raw_vec.rs @@ -904,11 +904,12 @@ mod tests { let cap = v.cap(); v.reserve(cap, 4096 * 32); assert!(v.cap() >= cap + 4096 * 32 - && v.cap() <= cap + 4096 * 34); + && v.cap() <= cap + 4096 * 40); // large vectors grow with a growth-factor equals 2x: let cap = v.cap(); v.reserve(cap, 1); + assert_eq!(v.cap(), cap * 2); assert!(v.cap() >= cap * 2 && v.cap() <= cap * 2 + 4096); } } From 088127aeb508ae8fd281f225d1da99f64eb72bcd Mon Sep 17 00:00:00 2001 From: gnzlbg Date: Mon, 23 Oct 2017 21:19:11 +0200 Subject: [PATCH 08/12] old heuristic for empty vectors --- src/liballoc/raw_vec.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/liballoc/raw_vec.rs b/src/liballoc/raw_vec.rs index 603bea21e9aa7..cd173e4df7b29 100644 --- a/src/liballoc/raw_vec.rs +++ b/src/liballoc/raw_vec.rs @@ -210,16 +210,18 @@ impl RawVec { #[inline(always)] fn amortized_new_capacity(elem_size: usize, current_capacity: usize, capacity_increase: usize) -> usize { - assert!(elem_size != 0, "RawVecs of zero-sized types can't grow"); - // Computes the capacity from the `current_capacity` following the // growth-strategy. The function `alloc_guard` ensures that // `current_capacity <= isize::MAX` so that `current_capacity * N` where `N // <= 2` cannot overflow. let growth_capacity = match current_capacity { // Empty vector => at least 64 bytes - // [OLD]: 0 => if elem_size > (!0) / 8 { 1 } else { 4 }, - 0 => (64 / elem_size).max(1), + // + // [OLD]: + 0 => if elem_size > (!0) / 8 { 1 } else { 4 }, + // [NEW]: + // 0 => (64 / elem_size).max(1), + // // Small and large vectors (<= 4096 bytes, and >= 4096 * 32 bytes): // // FIXME: jemalloc specific behavior, allocators should provide a @@ -839,7 +841,7 @@ mod tests { v.reserve(50, 150); // (causes a realloc, thus using 50 + 150 = 200 units of fuel) assert_eq!(v.a.fuel, 250); } - +/* #[test] fn amortized_new_capacity_tests() { // empty vector: @@ -913,4 +915,5 @@ mod tests { assert!(v.cap() >= cap * 2 && v.cap() <= cap * 2 + 4096); } } + */ } From f9e2dce322da60509caa66e3105a9ccf9fa58d3d Mon Sep 17 00:00:00 2001 From: gnzlbg Date: Tue, 24 Oct 2017 15:30:25 +0200 Subject: [PATCH 09/12] tiny_alloc_4_e_4_elems_or_at_least_32_bytes --- src/liballoc/raw_vec.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/liballoc/raw_vec.rs b/src/liballoc/raw_vec.rs index cd173e4df7b29..74d561f3cbe69 100644 --- a/src/liballoc/raw_vec.rs +++ b/src/liballoc/raw_vec.rs @@ -218,9 +218,11 @@ fn amortized_new_capacity(elem_size: usize, current_capacity: usize, // Empty vector => at least 64 bytes // // [OLD]: - 0 => if elem_size > (!0) / 8 { 1 } else { 4 }, + // 0 => if elem_size > (!0) / 8 { 1 } else { 4 }, // [NEW]: // 0 => (64 / elem_size).max(1), + // [NEW 2]: + 0 => (32 / elem_size).max(4), // // Small and large vectors (<= 4096 bytes, and >= 4096 * 32 bytes): // From 42e751a48e771731e453abaa4bdda362dc179d83 Mon Sep 17 00:00:00 2001 From: gnzlbg Date: Wed, 25 Oct 2017 09:02:42 +0200 Subject: [PATCH 10/12] original_tiny_original_2x --- src/liballoc/raw_vec.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/liballoc/raw_vec.rs b/src/liballoc/raw_vec.rs index 74d561f3cbe69..85d05ad135156 100644 --- a/src/liballoc/raw_vec.rs +++ b/src/liballoc/raw_vec.rs @@ -217,13 +217,14 @@ fn amortized_new_capacity(elem_size: usize, current_capacity: usize, let growth_capacity = match current_capacity { // Empty vector => at least 64 bytes // - // [OLD]: - // 0 => if elem_size > (!0) / 8 { 1 } else { 4 }, + // [Original]: + 0 => if elem_size > (!0) / 8 { 1 } else { 4 }, // [NEW]: // 0 => (64 / elem_size).max(1), // [NEW 2]: - 0 => (32 / elem_size).max(4), + // 0 => (32 / elem_size).max(4), // + // Small and large vectors (<= 4096 bytes, and >= 4096 * 32 bytes): // // FIXME: jemalloc specific behavior, allocators should provide a @@ -231,10 +232,14 @@ fn amortized_new_capacity(elem_size: usize, current_capacity: usize, // // jemalloc can never grow in place small blocks but blocks larger // than or equal to 4096 bytes can be expanded in place: - c if c < 4096 / elem_size => 2 * c, - c if c > 4096 * 32 / elem_size => 2 * c, + // c if c < 4096 / elem_size => 2 * c, + // c if c > 4096 * 32 / elem_size => 2 * c, + // Medium sized vectors in the [4096, 4096 * 32) bytes range: - c => c / 2 * 3 + // c => c / 2 * 3, + + // [Original] + c => 2 * c, }; cmp::max(growth_capacity, current_capacity.checked_add(capacity_increase).unwrap()) From 3639c161fb4f9ca802e32155731d687b7a8a7e43 Mon Sep 17 00:00:00 2001 From: gnzlbg Date: Wed, 25 Oct 2017 20:35:32 +0200 Subject: [PATCH 11/12] original_tiny_1.5x_growth_otherwise --- src/liballoc/raw_vec.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/liballoc/raw_vec.rs b/src/liballoc/raw_vec.rs index 85d05ad135156..316aa7c7a7298 100644 --- a/src/liballoc/raw_vec.rs +++ b/src/liballoc/raw_vec.rs @@ -236,10 +236,10 @@ fn amortized_new_capacity(elem_size: usize, current_capacity: usize, // c if c > 4096 * 32 / elem_size => 2 * c, // Medium sized vectors in the [4096, 4096 * 32) bytes range: - // c => c / 2 * 3, + c => c / 2 * 3 + 1, // [Original] - c => 2 * c, + // c => 2 * c, }; cmp::max(growth_capacity, current_capacity.checked_add(capacity_increase).unwrap()) From 18a82aa633dddbf5cc2f0b04e84c8ce593e82545 Mon Sep 17 00:00:00 2001 From: gnzlbg Date: Fri, 10 Nov 2017 10:56:14 +0100 Subject: [PATCH 12/12] use alloc_excess and realloc_excess --- src/doc/book | 2 +- src/doc/nomicon | 2 +- src/liballoc/raw_vec.rs | 93 +++++++++++++++++------------------------ src/liblibc | 2 +- src/tools/cargo | 2 +- src/tools/clippy | 2 +- src/tools/rls | 2 +- src/tools/rustfmt | 2 +- 8 files changed, 45 insertions(+), 62 deletions(-) diff --git a/src/doc/book b/src/doc/book index 7db393dae740d..08e79609ce885 160000 --- a/src/doc/book +++ b/src/doc/book @@ -1 +1 @@ -Subproject commit 7db393dae740d84775b73f403123c866e94e3a5b +Subproject commit 08e79609ce88583fa7286157dfe497486a09fabe diff --git a/src/doc/nomicon b/src/doc/nomicon index 1625e0b8c8708..0ee3f7265e9d0 160000 --- a/src/doc/nomicon +++ b/src/doc/nomicon @@ -1 +1 @@ -Subproject commit 1625e0b8c870891b84b0969777a974bf87be579b +Subproject commit 0ee3f7265e9d09746d901cef6f1f300baff1d923 diff --git a/src/liballoc/raw_vec.rs b/src/liballoc/raw_vec.rs index 316aa7c7a7298..1f159bb36b187 100644 --- a/src/liballoc/raw_vec.rs +++ b/src/liballoc/raw_vec.rs @@ -13,7 +13,7 @@ use core::mem; use core::ops::Drop; use core::ptr::{self, Unique}; use core::slice; -use heap::{Alloc, Layout, Heap}; +use heap::{Alloc, Layout, Heap, Excess}; use super::boxed::Box; /// A low-level utility for more ergonomically allocating, reallocating, and deallocating @@ -174,29 +174,29 @@ impl RawVec { /// Panics on overflow if `current_capacity + capacity_increase > /// std::usize::MAX`. /// -/// /// # Growth strategy /// /// RawVec grows differently depending on: /// -/// - 1. initial size: grows from zero to at least 64 bytes; +/// - 1. initial size: grows from zero to four elements or at least 64 bytes; /// use `with_capacity` to avoid a growth from zero. /// /// - 2. vector size: -/// - small vectors (<= 4096 bytes) and large vectors (>= 4096 * 32 bytes) -/// grow with a growth factor of 2x. -/// - otherwise (medium-sized vectors) grow with a growth factor of 1.5x. +/// - small vectors (<= 4096 bytes) grow with a factor of 2x. +/// - medium-sized and large vectors grow with a growth factor of 1.5x. /// /// # Growth factor /// -/// Medium-sized vectors' growth-factor is chosen to allow reusing memory from -/// previous allocations. Previously freed memory can be reused after +/// The growth factor of medium-sized and large vectors is chosen to allow +/// reusing memory from previously-freed allocations in subsequent allocations. +/// +/// Depending on the growth factor, previously freed memory can be reused after /// -/// - 4 reallocations for a growth factor of 1.5x -/// - 3 reallocations for a growth factor of 1.45x -/// - 2 reallocations for a growth factor of 1.3x +/// - 4 reallocations for a growth factor of 1.5x, +/// - 3 reallocations for a growth factor of 1.45x, +/// - 2 reallocations for a growth factor of 1.3x, /// -/// Which one is better [is application +/// Which growth-factor is better [is application /// dependent](https://stackoverflow.com/questions/1100311/ /// what-is-the-ideal-growth-rate-for-a-dynamically-allocated-array), /// also some claim that [the golden ration (1.618) is @@ -205,7 +205,12 @@ impl RawVec { /// The trade-off is having to wait for many reallocations to be able to /// reuse old memory. /// -/// Note: a factor of 2x _never_ allows reusing previously-freed memory. +/// A factor of 2x _never_ allows reusing any previously-freed memory but +/// `jemalloc`'s memory blocks < 4096 bytes cannot grow in place, that is, +/// trying to grow a vector < 4096 bytes is always going to require allocating +/// new memory and copying the contents over. Since `jemalloc`'s memory pools +/// for small allocations grow with powers of 2 it makes sense to keep a +/// growth-factor of 2x for these allocations. /// #[inline(always)] fn amortized_new_capacity(elem_size: usize, current_capacity: usize, @@ -215,31 +220,18 @@ fn amortized_new_capacity(elem_size: usize, current_capacity: usize, // `current_capacity <= isize::MAX` so that `current_capacity * N` where `N // <= 2` cannot overflow. let growth_capacity = match current_capacity { - // Empty vector => at least 64 bytes - // - // [Original]: - 0 => if elem_size > (!0) / 8 { 1 } else { 4 }, - // [NEW]: - // 0 => (64 / elem_size).max(1), - // [NEW 2]: - // 0 => (32 / elem_size).max(4), - // - - // Small and large vectors (<= 4096 bytes, and >= 4096 * 32 bytes): - // - // FIXME: jemalloc specific behavior, allocators should provide a - // way to query the byte size of blocks that can grow inplace. - // - // jemalloc can never grow in place small blocks but blocks larger - // than or equal to 4096 bytes can be expanded in place: - // c if c < 4096 / elem_size => 2 * c, - // c if c > 4096 * 32 / elem_size => 2 * c, - - // Medium sized vectors in the [4096, 4096 * 32) bytes range: - c => c / 2 * 3 + 1, - - // [Original] - // c => 2 * c, + // Empty vector => 4 elements or at least 64 bytes + 0 => (64 / elem_size).max(4), + + // Small vectors: jemalloc cannot grow in place blocks smaller than 4096 + // bytes so until then the memory of previously-freed allocations + // cannot be reused by subsequent allocations: + c if c < 4096 / elem_size => 2 * c, + + // Medium and large vectors (>= 4096 bytes): a growth factor of 1.5 + // allows the memory of a previously-freeed allocation to be reused + // after 4 subsequent allocations: + c => (c / 2 + 1) * 3, }; cmp::max(growth_capacity, current_capacity.checked_add(capacity_increase).unwrap()) @@ -592,31 +584,25 @@ impl RawVec { used_cap, needed_extra_cap); - let new_layout = match Layout::array::(new_cap) { - Some(layout) => layout, - None => panic!("capacity overflow"), - }; - let (_, usable_size) = self.a.usable_size(&new_layout); - let new_cap = usable_size / elem_size; let new_layout = match Layout::array::(new_cap) { Some(layout) => layout, None => panic!("capacity overflow"), }; // FIXME: may crash and burn on over-reserve alloc_guard(new_layout.size()); - let res = match self.current_layout() { + let Excess(res, usable_size) = match self.current_layout() { Some(layout) => { let old_ptr = self.ptr.as_ptr() as *mut u8; - self.a.realloc(old_ptr, layout, new_layout) + self.a.realloc_excess(old_ptr, layout, new_layout) } - None => self.a.alloc(new_layout), + None => self.a.alloc_excess(new_layout), }; let uniq = match res { Ok(ptr) => Unique::new_unchecked(ptr as *mut T), Err(e) => self.a.oom(e), }; self.ptr = uniq; - self.cap = new_cap; + self.cap = usable_size / mem::size_of::(); } } @@ -666,17 +652,14 @@ impl RawVec { let ptr = self.ptr() as *mut _; let new_layout = Layout::new::().repeat(new_cap).unwrap().0; - let (_, usable_size) = self.a.usable_size(&new_layout); - let new_cap = usable_size / elem_size; - let new_layout = match Layout::array::(new_cap) { - Some(layout) => layout, - None => panic!("capacity overflow"), - }; // FIXME: may crash and burn on over-reserve alloc_guard(new_layout.size()); match self.a.grow_in_place(ptr, old_layout, new_layout) { Ok(_) => { - self.cap = new_cap; + // FIXME: grow_in_place swallows the usable_size, so we + // need to recompute it here: + let (_, new_usable_size) = self.a.usable_size(&new_layout); + self.cap = new_usable_size / mem::size_of::(); true } Err(_) => { diff --git a/src/liblibc b/src/liblibc index 68f9959e53da6..44e4018e1a377 160000 --- a/src/liblibc +++ b/src/liblibc @@ -1 +1 @@ -Subproject commit 68f9959e53da6c70bed7119cd09342859d431266 +Subproject commit 44e4018e1a37716286ec98cb5b7dd7d33ecaf940 diff --git a/src/tools/cargo b/src/tools/cargo index b83550edc300e..e447ac7e94b7f 160000 --- a/src/tools/cargo +++ b/src/tools/cargo @@ -1 +1 @@ -Subproject commit b83550edc300e3d80dd16d0440123ffc1ad77bb9 +Subproject commit e447ac7e94b7f56ab13e361f9e324dafe3eb0a34 diff --git a/src/tools/clippy b/src/tools/clippy index f76225e388717..b62b1b68edcdf 160000 --- a/src/tools/clippy +++ b/src/tools/clippy @@ -1 +1 @@ -Subproject commit f76225e3887170743403af9204887918b5db5a80 +Subproject commit b62b1b68edcdf23a70cb12f31403c80e97f13634 diff --git a/src/tools/rls b/src/tools/rls index 9ad92e134ff56..93b47d14cef57 160000 --- a/src/tools/rls +++ b/src/tools/rls @@ -1 +1 @@ -Subproject commit 9ad92e134ff56df52481cf19dc3da14b9e735061 +Subproject commit 93b47d14cef5720bba7cfb4dcb8078fbf1f706c1 diff --git a/src/tools/rustfmt b/src/tools/rustfmt index 51b03c3aaf5e6..16a478368c8dc 160000 --- a/src/tools/rustfmt +++ b/src/tools/rustfmt @@ -1 +1 @@ -Subproject commit 51b03c3aaf5e69afbb7508e566c5da2bf0bc3662 +Subproject commit 16a478368c8dcc0c0ee47372a9f663b23d28b097