Auto merge of #50235 - Zoxc:rayon, r=michaelwoerister

Add a Rayon thread pool

r? @michaelwoerister

Author: bors

src/Cargo.lock

@@ -88,3 +88,33 @@ impl<'v, 'hir, V> ItemLikeVisitor<'hir> for DeepVisitor<'v, V>
         self.visitor.visit_impl_item(impl_item);
     }
 }
+
+/// A parallel variant of ItemLikeVisitor
+pub trait ParItemLikeVisitor<'hir> {
+    fn visit_item(&self, item: &'hir Item);
+    fn visit_trait_item(&self, trait_item: &'hir TraitItem);
+    fn visit_impl_item(&self, impl_item: &'hir ImplItem);
+}
+
+pub trait IntoVisitor<'hir> {
+    type Visitor: Visitor<'hir>;
+    fn into_visitor(&self) -> Self::Visitor;
+}
+
+pub struct ParDeepVisitor<V>(pub V);
+
+impl<'hir, V> ParItemLikeVisitor<'hir> for ParDeepVisitor<V>
+    where V: IntoVisitor<'hir>
+{
+    fn visit_item(&self, item: &'hir Item) {
+        self.0.into_visitor().visit_item(item);
+    }
+
+    fn visit_trait_item(&self, trait_item: &'hir TraitItem) {
+        self.0.into_visitor().visit_trait_item(trait_item);
+    }
+
+    fn visit_impl_item(&self, impl_item: &'hir ImplItem) {
+        self.0.into_visitor().visit_impl_item(impl_item);
+    }
+}

src/librustc/hir/itemlikevisit.rs

@@ -48,6 +48,7 @@ use ty::AdtKind;
 use ty::maps::Providers;
 
 use rustc_data_structures::indexed_vec;
+use rustc_data_structures::sync::{ParallelIterator, par_iter, Send, Sync, scope};
 
 use serialize::{self, Encoder, Encodable, Decoder, Decodable};
 use std::collections::BTreeMap;
@@ -720,6 +721,31 @@ impl Crate {
         }
     }
 
+    /// A parallel version of visit_all_item_likes
+    pub fn par_visit_all_item_likes<'hir, V>(&'hir self, visitor: &V)
+        where V: itemlikevisit::ParItemLikeVisitor<'hir> + Sync + Send
+    {
+        scope(|s| {
+            s.spawn(|_| {
+                par_iter(&self.items).for_each(|(_, item)| {
+                    visitor.visit_item(item);
+                });
+            });
+
+            s.spawn(|_| {
+                par_iter(&self.trait_items).for_each(|(_, trait_item)| {
+                    visitor.visit_trait_item(trait_item);
+                });
+            });
+
+            s.spawn(|_| {
+                par_iter(&self.impl_items).for_each(|(_, impl_item)| {
+                    visitor.visit_impl_item(impl_item);
+                });
+            });
+        });
+    }
+
     pub fn body(&self, id: BodyId) -> &Body {
         &self.bodies[&id]
     }

src/librustc/hir/mod.rs

@@ -38,7 +38,7 @@ use syntax::ext::base::SyntaxExtension;
 use syntax::symbol::Symbol;
 use syntax_pos::Span;
 use rustc_target::spec::Target;
-use rustc_data_structures::sync::{MetadataRef, Lrc};
+use rustc_data_structures::sync::{self, MetadataRef, Lrc};
 
 pub use self::NativeLibraryKind::*;
 
@@ -255,6 +255,8 @@ pub trait CrateStore {
     fn metadata_encoding_version(&self) -> &[u8];
 }
 
+pub type CrateStoreDyn = CrateStore + sync::Sync;
+
 // FIXME: find a better place for this?
 pub fn validate_crate_name(sess: Option<&Session>, s: &str, sp: Option<Span>) {
     let mut err_count = 0;

src/librustc/middle/cstore.rs

@@ -869,10 +869,16 @@ impl Session {
         ret
     }
 
+    /// Returns the number of query threads that should be used for this
+    /// compilation
+    pub fn query_threads_from_opts(opts: &config::Options) -> usize {
+        opts.debugging_opts.query_threads.unwrap_or(1)
+    }
+
     /// Returns the number of query threads that should be used for this
     /// compilation
     pub fn query_threads(&self) -> usize {
-        self.opts.debugging_opts.query_threads.unwrap_or(1)
+        Self::query_threads_from_opts(&self.opts)
     }
 
     /// Returns the number of codegen units that should be used for this

src/librustc/session/mod.rs

@@ -26,7 +26,7 @@ use lint::{self, Lint};
 use ich::{StableHashingContext, NodeIdHashingMode};
 use infer::canonical::{CanonicalVarInfo, CanonicalVarInfos};
 use infer::outlives::free_region_map::FreeRegionMap;
-use middle::cstore::{CrateStore, LinkMeta};
+use middle::cstore::{CrateStoreDyn, LinkMeta};
 use middle::cstore::EncodedMetadata;
 use middle::lang_items;
 use middle::resolve_lifetime::{self, ObjectLifetimeDefault};
@@ -852,7 +852,7 @@ pub struct GlobalCtxt<'tcx> {
     global_arenas: &'tcx GlobalArenas<'tcx>,
     global_interners: CtxtInterners<'tcx>,
 
-    cstore: &'tcx dyn CrateStore,
+    cstore: &'tcx CrateStoreDyn,
 
     pub sess: &'tcx Session,
 
@@ -1188,7 +1188,7 @@ impl<'a, 'gcx, 'tcx> TyCtxt<'a, 'gcx, 'tcx> {
     /// value (types, substs, etc.) can only be used while `ty::tls` has a valid
     /// reference to the context, to allow formatting values that need it.
     pub fn create_and_enter<F, R>(s: &'tcx Session,
-                                  cstore: &'tcx dyn CrateStore,
+                                  cstore: &'tcx CrateStoreDyn,
                                   local_providers: ty::maps::Providers<'tcx>,
                                   extern_providers: ty::maps::Providers<'tcx>,
                                   arenas: &'tcx AllArenas<'tcx>,
@@ -1800,9 +1800,11 @@ pub mod tls {
     /// in librustc otherwise. It is used to when diagnostic messages are
     /// emitted and stores them in the current query, if there is one.
     fn track_diagnostic(diagnostic: &Diagnostic) {
-        with_context(|context| {
-            if let Some(ref query) = context.query {
-                query.diagnostics.lock().push(diagnostic.clone());
+        with_context_opt(|icx| {
+            if let Some(icx) = icx {
+                if let Some(ref query) = icx.query {
+                    query.diagnostics.lock().push(diagnostic.clone());
+                }
             }
         })
     }

src/librustc/ty/context.rs

@@ -16,6 +16,7 @@ serialize = { path = "../libserialize" }
 cfg-if = "0.1.2"
 stable_deref_trait = "1.0.0"
 parking_lot_core = "0.2.8"
+rustc-rayon = "0.1.0"
 
 [dependencies.parking_lot]
 version = "0.5"

src/librustc_data_structures/Cargo.toml

@@ -44,6 +44,7 @@ extern crate parking_lot;
 #[macro_use]
 extern crate cfg_if;
 extern crate stable_deref_trait;
+extern crate rustc_rayon as rayon;
 
 // See librustc_cratesio_shim/Cargo.toml for a comment explaining this.
 #[allow(unused_extern_crates)]

src/librustc_data_structures/lib.rs

@@ -8,7 +8,7 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-//! This mdoule defines types which are thread safe if cfg!(parallel_queries) is true.
+//! This module defines types which are thread safe if cfg!(parallel_queries) is true.
 //!
 //! `Lrc` is an alias of either Rc or Arc.
 //!
@@ -40,6 +40,29 @@ use std;
 use std::ops::{Deref, DerefMut};
 use owning_ref::{Erased, OwningRef};
 
+pub fn serial_join<A, B, RA, RB>(oper_a: A, oper_b: B) -> (RA, RB)
+    where A: FnOnce() -> RA,
+          B: FnOnce() -> RB
+{
+    (oper_a(), oper_b())
+}
+
+pub struct SerialScope;
+
+impl SerialScope {
+    pub fn spawn<F>(&self, f: F)
+        where F: FnOnce(&SerialScope)
+    {
+        f(self)
+    }
+}
+
+pub fn serial_scope<F, R>(f: F) -> R
+    where F: FnOnce(&SerialScope) -> R
+{
+    f(&SerialScope)
+}
+
 cfg_if! {
     if #[cfg(not(parallel_queries))] {
         pub auto trait Send {}
@@ -55,9 +78,19 @@ cfg_if! {
             }
         }
 
+        pub use self::serial_join as join;
+        pub use self::serial_scope as scope;
+
+        pub use std::iter::Iterator as ParallelIterator;
+
+        pub fn par_iter<T: IntoIterator>(t: T) -> T::IntoIter {
+            t.into_iter()
+        }
+
         pub type MetadataRef = OwningRef<Box<Erased>, [u8]>;
 
         pub use std::rc::Rc as Lrc;
+        pub use std::rc::Weak as Weak;
         pub use std::cell::Ref as ReadGuard;
         pub use std::cell::RefMut as WriteGuard;
         pub use std::cell::RefMut as LockGuard;
@@ -160,13 +193,22 @@ cfg_if! {
         pub use parking_lot::MutexGuard as LockGuard;
 
         pub use std::sync::Arc as Lrc;
+        pub use std::sync::Weak as Weak;
 
         pub use self::Lock as MTLock;
 
         use parking_lot::Mutex as InnerLock;
         use parking_lot::RwLock as InnerRwLock;
 
         use std::thread;
+        pub use rayon::{join, scope};
+
+        pub use rayon::iter::ParallelIterator;
+        use rayon::iter::IntoParallelIterator;
+
+        pub fn par_iter<T: IntoParallelIterator>(t: T) -> T::Iter {
+            t.into_par_iter()
+        }
 
         pub type MetadataRef = OwningRef<Box<Erased + Send + Sync>, [u8]>;
 

src/librustc_data_structures/sync.rs

@@ -13,6 +13,8 @@ arena = { path = "../libarena" }
 graphviz = { path = "../libgraphviz" }
 log = "0.4"
 env_logger = { version = "0.5", default-features = false }
+rustc-rayon = "0.1.0"
+scoped-tls = { version = "0.1.1", features = ["nightly"] }
 rustc = { path = "../librustc" }
 rustc_allocator = { path = "../librustc_allocator" }
 rustc_target = { path = "../librustc_target" }