Auto merge of rust-lang#37270 - Mark-Simulacrum:smallvec-optimized-arenas, r=eddyb

Add ArrayVec and AccumulateVec to reduce heap allocations during interning of slices

Updates `mk_tup`, `mk_type_list`, and `mk_substs` to allow interning directly from iterators. The previous PR, rust-lang#37220, changed some of the calls to pass a borrowed slice from `Vec` instead of directly passing the iterator, and these changes further optimize that to avoid the allocation entirely.

This change yields 50% less malloc calls in [some cases](https://pastebin.mozilla.org/8921686). It also yields decent, though not amazing, performance improvements:
```
futures-rs-test  4.091s vs  4.021s --> 1.017x faster (variance: 1.004x, 1.004x)
helloworld       0.219s vs  0.220s --> 0.993x faster (variance: 1.010x, 1.018x)
html5ever-2016-  3.805s vs  3.736s --> 1.018x faster (variance: 1.003x, 1.009x)
hyper.0.5.0      4.609s vs  4.571s --> 1.008x faster (variance: 1.015x, 1.017x)
inflate-0.1.0    3.864s vs  3.883s --> 0.995x faster (variance: 1.232x, 1.005x)
issue-32062-equ  0.309s vs  0.299s --> 1.033x faster (variance: 1.014x, 1.003x)
issue-32278-big  1.614s vs  1.594s --> 1.013x faster (variance: 1.007x, 1.004x)
jld-day15-parse  1.390s vs  1.326s --> 1.049x faster (variance: 1.006x, 1.009x)
piston-image-0. 10.930s vs 10.675s --> 1.024x faster (variance: 1.006x, 1.010x)
reddit-stress    2.302s vs  2.261s --> 1.019x faster (variance: 1.010x, 1.026x)
regex.0.1.30     2.250s vs  2.240s --> 1.005x faster (variance: 1.087x, 1.011x)
rust-encoding-0  1.895s vs  1.887s --> 1.005x faster (variance: 1.005x, 1.018x)
syntex-0.42.2   29.045s vs 28.663s --> 1.013x faster (variance: 1.004x, 1.006x)
syntex-0.42.2-i 13.925s vs 13.868s --> 1.004x faster (variance: 1.022x, 1.007x)
```

We implement a small-size optimized vector, intended to be used primarily for collection of presumed to be short iterators. This vector cannot be "upsized/reallocated" into a heap-allocated vector, since that would require (slow) branching logic, but during the initial collection from an iterator heap-allocation is possible.

We make the new `AccumulateVec` and `ArrayVec` generic over implementors of the `Array` trait, of which there is currently one, `[T; 8]`. In the future, this is likely to expand to other values of N.

Huge thanks to @nnethercote for collecting the performance and other statistics mentioned above.

Author: bors

src/libarena/lib.rs

@@ -157,13 +157,15 @@ impl<T> TypedArena<T> {
 
     /// Allocates a slice of objects that are copy into the `TypedArena`, returning a mutable
     /// reference to it. Will panic if passed a zero-sized types.
+    ///
+    /// Panics:
+    ///  - Zero-sized types
+    ///  - Zero-length slices
     #[inline]
     pub fn alloc_slice(&self, slice: &[T]) -> &mut [T]
         where T: Copy {
         assert!(mem::size_of::<T>() != 0);
-        if slice.len() == 0 {
-            return unsafe { slice::from_raw_parts_mut(heap::EMPTY as *mut T, 0) };
-        }
+        assert!(slice.len() != 0);
 
         let available_capacity_bytes = self.end.get() as usize - self.ptr.get() as usize;
         let at_least_bytes = slice.len() * mem::size_of::<T>();

src/libcore/result.rs

@@ -1008,6 +1008,11 @@ impl<A, E, V: FromIterator<A>> FromIterator<Result<A, E>> for Result<V, E> {
                     None => None,
                 }
             }
+
+            fn size_hint(&self) -> (usize, Option<usize>) {
+                let (_min, max) = self.iter.size_hint();
+                (0, max)
+            }
         }
 
         let mut adapter = Adapter { iter: iter.into_iter(), err: None };

src/librustc/mir/tcx.rs

@@ -163,7 +163,7 @@ impl<'tcx> Rvalue<'tcx> {
                 let lhs_ty = lhs.ty(mir, tcx);
                 let rhs_ty = rhs.ty(mir, tcx);
                 let ty = op.ty(tcx, lhs_ty, rhs_ty);
-                let ty = tcx.mk_tup(&[ty, tcx.types.bool]);
+                let ty = tcx.intern_tup(&[ty, tcx.types.bool]);
                 Some(ty)
             }
             &Rvalue::UnaryOp(_, ref operand) => {
@@ -184,7 +184,7 @@ impl<'tcx> Rvalue<'tcx> {
                     }
                     AggregateKind::Tuple => {
                         Some(tcx.mk_tup(
-                            &ops.iter().map(|op| op.ty(mir, tcx)).collect::<Vec<_>>()
+                            ops.iter().map(|op| op.ty(mir, tcx))
                         ))
                     }
                     AggregateKind::Adt(def, _, substs, _) => {

src/librustc/traits/fulfill.rs

@@ -11,7 +11,7 @@
 use dep_graph::DepGraph;
 use infer::{InferCtxt, InferOk};
 use ty::{self, Ty, TypeFoldable, ToPolyTraitRef, TyCtxt, ToPredicate};
-use ty::subst::{Substs, Subst};
+use ty::subst::Subst;
 use rustc_data_structures::obligation_forest::{ObligationForest, Error};
 use rustc_data_structures::obligation_forest::{ForestObligation, ObligationProcessor};
 use std::marker::PhantomData;
@@ -159,7 +159,7 @@ impl<'a, 'gcx, 'tcx> DeferredObligation<'tcx> {
                 let concrete_ty = ty_scheme.ty.subst(tcx, substs);
                 let predicate = ty::TraitRef {
                     def_id: self.predicate.def_id(),
-                    substs: Substs::new_trait(tcx, concrete_ty, &[])
+                    substs: tcx.mk_substs_trait(concrete_ty, &[])
                 }.to_predicate();
 
                 let original_obligation = Obligation::new(self.cause.clone(),

src/librustc/traits/select.rs

@@ -2596,7 +2596,7 @@ impl<'cx, 'gcx, 'tcx> SelectionContext<'cx, 'gcx, 'tcx> {
                         k
                     }
                 });
-                let substs = Substs::new(tcx, params);
+                let substs = tcx.mk_substs(params);
                 for &ty in fields.split_last().unwrap().1 {
                     if ty.subst(tcx, substs).references_error() {
                         return Err(Unimplemented);
@@ -2616,7 +2616,7 @@ impl<'cx, 'gcx, 'tcx> SelectionContext<'cx, 'gcx, 'tcx> {
                         k
                     }
                 });
-                let new_struct = tcx.mk_adt(def, Substs::new(tcx, params));
+                let new_struct = tcx.mk_adt(def, tcx.mk_substs(params));
                 let origin = TypeOrigin::Misc(obligation.cause.span);
                 let InferOk { obligations, .. } =
                     self.infcx.sub_types(false, origin, new_struct, target)

src/librustc/traits/util.rs

@@ -380,7 +380,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
             Ok(def_id) => {
                 Ok(ty::TraitRef {
                     def_id: def_id,
-                    substs: Substs::new_trait(self, param_ty, &[])
+                    substs: self.mk_substs_trait(param_ty, &[])
                 })
             }
             Err(e) => {
@@ -400,7 +400,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
     {
         let trait_ref = ty::TraitRef {
             def_id: trait_def_id,
-            substs: Substs::new_trait(self, param_ty, ty_params)
+            substs: self.mk_substs_trait(param_ty, ty_params)
         };
         predicate_for_trait_ref(cause, trait_ref, recursion_depth)
     }
@@ -486,11 +486,11 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
     {
         let arguments_tuple = match tuple_arguments {
             TupleArgumentsFlag::No => sig.0.inputs[0],
-            TupleArgumentsFlag::Yes => self.mk_tup(&sig.0.inputs),
+            TupleArgumentsFlag::Yes => self.intern_tup(&sig.0.inputs[..]),
         };
         let trait_ref = ty::TraitRef {
             def_id: fn_trait_def_id,
-            substs: Substs::new_trait(self, self_ty, &[arguments_tuple]),
+            substs: self.mk_substs_trait(self_ty, &[arguments_tuple]),
         };
         ty::Binder((trait_ref, sig.0.output))
     }

src/librustc/ty/context.rs

@@ -36,6 +36,7 @@ use ty::maps;
 use util::common::MemoizationMap;
 use util::nodemap::{NodeMap, NodeSet, DefIdMap, DefIdSet};
 use util::nodemap::{FnvHashMap, FnvHashSet};
+use rustc_data_structures::accumulate_vec::AccumulateVec;
 
 use arena::TypedArena;
 use std::borrow::Borrow;
@@ -44,6 +45,7 @@ use std::hash::{Hash, Hasher};
 use std::mem;
 use std::ops::Deref;
 use std::rc::Rc;
+use std::iter;
 use syntax::ast::{self, Name, NodeId};
 use syntax::attr;
 use syntax::parse::token::{self, keywords};
@@ -824,6 +826,9 @@ impl<'a, 'tcx> Lift<'tcx> for Ty<'a> {
 impl<'a, 'tcx> Lift<'tcx> for &'a Substs<'a> {
     type Lifted = &'tcx Substs<'tcx>;
     fn lift_to_tcx<'b, 'gcx>(&self, tcx: TyCtxt<'b, 'gcx, 'tcx>) -> Option<&'tcx Substs<'tcx>> {
+        if self.len() == 0 {
+            return Some(Slice::empty());
+        }
         if let Some(&Interned(substs)) = tcx.interners.substs.borrow().get(&self[..]) {
             if *self as *const _ == substs as *const _ {
                 return Some(substs);
@@ -859,6 +864,9 @@ impl<'a, 'tcx> Lift<'tcx> for &'a Slice<Ty<'a>> {
     type Lifted = &'tcx Slice<Ty<'tcx>>;
     fn lift_to_tcx<'b, 'gcx>(&self, tcx: TyCtxt<'b, 'gcx, 'tcx>)
                              -> Option<&'tcx Slice<Ty<'tcx>>> {
+        if self.len() == 0 {
+            return Some(Slice::empty());
+        }
         if let Some(&Interned(list)) = tcx.interners.type_list.borrow().get(&self[..]) {
             if *self as *const _ == list as *const _ {
                 return Some(list);
@@ -1212,8 +1220,8 @@ macro_rules! slice_interners {
 }
 
 slice_interners!(
-    type_list: mk_type_list(Ty),
-    substs: mk_substs(Kind)
+    type_list: _intern_type_list(Ty),
+    substs: _intern_substs(Kind)
 );
 
 impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
@@ -1318,12 +1326,16 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
         self.mk_ty(TySlice(ty))
     }
 
-    pub fn mk_tup(self, ts: &[Ty<'tcx>]) -> Ty<'tcx> {
-        self.mk_ty(TyTuple(self.mk_type_list(ts)))
+    pub fn intern_tup(self, ts: &[Ty<'tcx>]) -> Ty<'tcx> {
+        self.mk_ty(TyTuple(self.intern_type_list(ts)))
+    }
+
+    pub fn mk_tup<I: InternAs<[Ty<'tcx>], Ty<'tcx>>>(self, iter: I) -> I::Output {
+        iter.intern_with(|ts| self.mk_ty(TyTuple(self.intern_type_list(ts))))
     }
 
     pub fn mk_nil(self) -> Ty<'tcx> {
-        self.mk_tup(&[])
+        self.intern_tup(&[])
     }
 
     pub fn mk_diverging_default(self) -> Ty<'tcx> {
@@ -1369,7 +1381,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
                       -> Ty<'tcx> {
         self.mk_closure_from_closure_substs(closure_id, ClosureSubsts {
             func_substs: substs,
-            upvar_tys: self.mk_type_list(tys)
+            upvar_tys: self.intern_type_list(tys)
         })
     }
 
@@ -1414,6 +1426,40 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
         self.mk_ty(TyAnon(def_id, substs))
     }
 
+    pub fn intern_type_list(self, ts: &[Ty<'tcx>]) -> &'tcx Slice<Ty<'tcx>> {
+        if ts.len() == 0 {
+            Slice::empty()
+        } else {
+            self._intern_type_list(ts)
+        }
+    }
+
+    pub fn intern_substs(self, ts: &[Kind<'tcx>]) -> &'tcx Slice<Kind<'tcx>> {
+        if ts.len() == 0 {
+            Slice::empty()
+        } else {
+            self._intern_substs(ts)
+        }
+    }
+
+    pub fn mk_type_list<I: InternAs<[Ty<'tcx>],
+                        &'tcx Slice<Ty<'tcx>>>>(self, iter: I) -> I::Output {
+        iter.intern_with(|xs| self.intern_type_list(xs))
+    }
+
+    pub fn mk_substs<I: InternAs<[Kind<'tcx>],
+                     &'tcx Slice<Kind<'tcx>>>>(self, iter: I) -> I::Output {
+        iter.intern_with(|xs| self.intern_substs(xs))
+    }
+
+    pub fn mk_substs_trait(self,
+                     s: Ty<'tcx>,
+                     t: &[Ty<'tcx>])
+                    -> &'tcx Substs<'tcx>
+    {
+        self.mk_substs(iter::once(s).chain(t.into_iter().cloned()).map(Kind::from))
+    }
+
     pub fn trait_items(self, trait_did: DefId) -> Rc<Vec<ty::ImplOrTraitItem<'gcx>>> {
         self.trait_items_cache.memoize(trait_did, || {
             let def_ids = self.impl_or_trait_items(trait_did);
@@ -1432,3 +1478,39 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
         })
     }
 }
+
+pub trait InternAs<T: ?Sized, R> {
+    type Output;
+    fn intern_with<F>(self, F) -> Self::Output
+        where F: FnOnce(&T) -> R;
+}
+
+impl<I, T, R, E> InternAs<[T], R> for I
+    where E: InternIteratorElement<T, R>,
+          I: Iterator<Item=E> {
+    type Output = E::Output;
+    fn intern_with<F>(self, f: F) -> Self::Output
+        where F: FnOnce(&[T]) -> R {
+        E::intern_with(self, f)
+    }
+}
+
+pub trait InternIteratorElement<T, R>: Sized {
+    type Output;
+    fn intern_with<I: Iterator<Item=Self>, F: FnOnce(&[T]) -> R>(iter: I, f: F) -> Self::Output;
+}
+
+impl<T, R> InternIteratorElement<T, R> for T {
+    type Output = R;
+    fn intern_with<I: Iterator<Item=Self>, F: FnOnce(&[T]) -> R>(iter: I, f: F) -> Self::Output {
+        f(&iter.collect::<AccumulateVec<[_; 8]>>())
+    }
+}
+
+impl<T, R, E> InternIteratorElement<T, R> for Result<T, E> {
+    type Output = Result<R, E>;
+    fn intern_with<I: Iterator<Item=Self>, F: FnOnce(&[T]) -> R>(iter: I, f: F) -> Self::Output {
+        Ok(f(&iter.collect::<Result<AccumulateVec<[_; 8]>, _>>()?))
+    }
+}
+

src/librustc/ty/mod.rs

@@ -41,6 +41,7 @@ use std::ops::Deref;
 use std::rc::Rc;
 use std::slice;
 use std::vec::IntoIter;
+use std::mem;
 use syntax::ast::{self, Name, NodeId};
 use syntax::attr;
 use syntax::parse::token::{self, InternedString};
@@ -561,6 +562,14 @@ impl<'a, T> IntoIterator for &'a Slice<T> {
 
 impl<'tcx> serialize::UseSpecializedDecodable for &'tcx Slice<Ty<'tcx>> {}
 
+impl<T> Slice<T> {
+    pub fn empty<'a>() -> &'a Slice<T> {
+        unsafe {
+            mem::transmute(slice::from_raw_parts(0x1 as *const T, 0))
+        }
+    }
+}
+
 /// Upvars do not get their own node-id. Instead, we use the pair of
 /// the original var id (that is, the root variable that is referenced
 /// by the upvar) and the id of the closure expression.
@@ -1798,7 +1807,7 @@ impl<'a, 'tcx> AdtDefData<'tcx, 'tcx> {
             _ if tys.references_error() => tcx.types.err,
             0 => tcx.types.bool,
             1 => tys[0],
-            _ => tcx.mk_tup(&tys)
+            _ => tcx.intern_tup(&tys[..])
         };
 
         match self.sized_constraint.get(dep_node()) {
@@ -1874,7 +1883,7 @@ impl<'a, 'tcx> AdtDefData<'tcx, 'tcx> {
                 };
                 let sized_predicate = Binder(TraitRef {
                     def_id: sized_trait,
-                    substs: Substs::new_trait(tcx, ty, &[])
+                    substs: tcx.mk_substs_trait(ty, &[])
                 }).to_predicate();
                 let predicates = tcx.lookup_predicates(self.did).predicates;
                 if predicates.into_iter().any(|p| p == sized_predicate) {
@@ -2125,7 +2134,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
     pub fn node_id_item_substs(self, id: NodeId) -> ItemSubsts<'gcx> {
         match self.tables.borrow().item_substs.get(&id) {
             None => ItemSubsts {
-                substs: Substs::empty(self.global_tcx())
+                substs: self.global_tcx().intern_substs(&[])
             },
             Some(ts) => ts.clone(),
         }
@@ -2797,7 +2806,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
         // regions, so it shouldn't matter what we use for the free id
         let free_id_outlive = self.region_maps.node_extent(ast::DUMMY_NODE_ID);
         ty::ParameterEnvironment {
-            free_substs: Substs::empty(self),
+            free_substs: self.intern_substs(&[]),
             caller_bounds: Vec::new(),
             implicit_region_bound: self.mk_region(ty::ReEmpty),
             free_id_outlive: free_id_outlive,

src/librustc/ty/relate.rs

@@ -49,12 +49,6 @@ pub trait TypeRelation<'a, 'gcx: 'a+'tcx, 'tcx: 'a> : Sized {
         Relate::relate(self, a, b)
     }
 
-    /// Relete elements of two slices pairwise.
-    fn relate_zip<T: Relate<'tcx>>(&mut self, a: &[T], b: &[T]) -> RelateResult<'tcx, Vec<T>> {
-        assert_eq!(a.len(), b.len());
-        a.iter().zip(b).map(|(a, b)| self.relate(a, b)).collect()
-    }
-
     /// Switch variance for the purpose of relating `a` and `b`.
     fn relate_with_variance<T: Relate<'tcx>>(&mut self,
                                              variance: ty::Variance,
@@ -158,7 +152,7 @@ pub fn relate_substs<'a, 'gcx, 'tcx, R>(relation: &mut R,
         }
     });
 
-    Substs::maybe_new(tcx, params)
+    Ok(tcx.mk_substs(params)?)
 }
 
 impl<'tcx> Relate<'tcx> for &'tcx ty::BareFnTy<'tcx> {
@@ -489,10 +483,7 @@ pub fn super_relate_tys<'a, 'gcx, 'tcx, R>(relation: &mut R,
         (&ty::TyTuple(as_), &ty::TyTuple(bs)) =>
         {
             if as_.len() == bs.len() {
-                let ts = as_.iter().zip(bs)
-                            .map(|(a, b)| relation.relate(a, b))
-                            .collect::<Result<Vec<_>, _>>()?;
-                Ok(tcx.mk_tup(&ts))
+                Ok(tcx.mk_tup(as_.iter().zip(bs).map(|(a, b)| relation.relate(a, b)))?)
             } else if !(as_.is_empty() || bs.is_empty()) {
                 Err(TypeError::TupleSize(
                     expected_found(relation, &as_.len(), &bs.len())))
@@ -544,10 +535,11 @@ impl<'tcx> Relate<'tcx> for ty::ClosureSubsts<'tcx> {
         where R: TypeRelation<'a, 'gcx, 'tcx>, 'gcx: 'a+'tcx, 'tcx: 'a
     {
         let substs = relate_substs(relation, None, a.func_substs, b.func_substs)?;
-        let upvar_tys = relation.relate_zip(&a.upvar_tys, &b.upvar_tys)?;
+        assert_eq!(a.upvar_tys.len(), b.upvar_tys.len());
         Ok(ty::ClosureSubsts {
             func_substs: substs,
-            upvar_tys: relation.tcx().mk_type_list(&upvar_tys)
+            upvar_tys: relation.tcx().mk_type_list(
+                a.upvar_tys.iter().zip(b.upvar_tys).map(|(a, b)| relation.relate(a, b)))?
         })
     }
 }

src/librustc/ty/structural_impls.rs

@@ -11,6 +11,7 @@
 use infer::type_variable;
 use ty::{self, Lift, Ty, TyCtxt};
 use ty::fold::{TypeFoldable, TypeFolder, TypeVisitor};
+use rustc_data_structures::accumulate_vec::AccumulateVec;
 
 use std::rc::Rc;
 use syntax::abi;
@@ -448,8 +449,8 @@ impl<'tcx> TypeFoldable<'tcx> for ty::TraitObject<'tcx> {
 
 impl<'tcx> TypeFoldable<'tcx> for &'tcx ty::Slice<Ty<'tcx>> {
     fn super_fold_with<'gcx: 'tcx, F: TypeFolder<'gcx, 'tcx>>(&self, folder: &mut F) -> Self {
-        let tys = self.iter().map(|t| t.fold_with(folder)).collect::<Vec<_>>();
-        folder.tcx().mk_type_list(&tys)
+        let v = self.iter().map(|t| t.fold_with(folder)).collect::<AccumulateVec<[_; 8]>>();
+        folder.tcx().intern_type_list(&v)
     }
 
     fn super_visit_with<V: TypeVisitor<'tcx>>(&self, visitor: &mut V) -> bool {