Refactor event type decoding and declaration (#221)

* Refactor event type decoding hand declartion

Fixes #196, #181, #28

## Dyanmic sized types

Before this change, the event decoder assume all the event types
have fixed sizes. Some counterexamples are: Hashes, AuthorityList.

In this change, instead of decoding by skipping the fixed-length bytes,
we introduce `type_segmenter` registry which decodes the raw event
bytes with the actual scale codec. So variable length types can be
handled correctly.

## New attribute for pallet type definition

In the past, trait associated type is the only way to add types to
the EventsDecoder implementation of a pallet. But in reality it's
common that the events in a pallet references some types not defined
in the trait associated types. Some examples are: `IdentificationTuple`
and `SessionIndex` in Session pallet.

In this change, we introduce more attributes to add the types:

```rust
#[module]
trait Pallet: System {
    #![event_type(SomeType)]
    #![event_alias(TypeNameAlias = SomeType)]
    #![event_alias(SomeOtherAlias = TypeWithAssociatedTypes<T>)]
}
```

## Tested

Compile with `nightly-2020-10-01`; smoke test to sync a full
Phala bockchain.

* Format code

* Make rustfmt::skip an outer attribute

* Ignore the sample code

* Alias the event segmenter closure

* Copy AuthorityList from sp_finality_grandpa

* Remove the unused static event type size

* Make segmenter as a trait, resue grandpa::Public

* Wrap PhantomData in struct TypeMarker

Author: h4x3rotab

proc-macro/src/module.rs

@@ -69,16 +69,62 @@ fn events_decoder_trait_name(module: &syn::Ident) -> syn::Ident {
 fn with_module_ident(module: &syn::Ident) -> syn::Ident {
     format_ident!("with_{}", module.to_string().to_snake_case())
 }
+
+type EventAttr = utils::UniAttr<syn::Type>;
+type EventAliasAttr = utils::UniAttr<utils::Attr<syn::Ident, syn::Type>>;
+
+/// Parses the event type definition macros within #[module]
+///
+/// It supports two ways to define the associated event type:
+///
+/// ```ignore
+/// #[module]
+/// trait Pallet: System {
+///     #![event_type(SomeType)]
+///     #![event_alias(TypeNameAlias = SomeType)]
+///     #![event_alias(SomeOtherAlias = TypeWithAssociatedTypes<T>)]
+/// }
+/// ```
+fn parse_event_type_attr(attr: &syn::Attribute) -> Option<(String, syn::Type)> {
+    let ident = utils::path_to_ident(&attr.path);
+    if ident == "event_type" {
+        let attrs: EventAttr = syn::parse2(attr.tokens.clone())
+            .map_err(|err| abort!("{}", err))
+            .unwrap();
+        let ty = attrs.attr;
+        let ident_str = quote!(#ty).to_string();
+        Some((ident_str, ty))
+    } else if ident == "event_alias" {
+        let attrs: EventAliasAttr = syn::parse2(attr.tokens.clone())
+            .map_err(|err| abort!("{}", err))
+            .unwrap();
+        let ty = attrs.attr.value;
+        let ident_str = attrs.attr.key.to_string();
+        Some((ident_str, ty))
+    } else {
+        None
+    }
+}
+
 /// Attribute macro that registers the type sizes used by the module; also sets the `MODULE` constant.
 pub fn module(_args: TokenStream, tokens: TokenStream) -> TokenStream {
     let input: Result<syn::ItemTrait, _> = syn::parse2(tokens.clone());
-    let input = if let Ok(input) = input {
+    let mut input = if let Ok(input) = input {
         input
     } else {
         // handle #[module(ignore)] by just returning the tokens
         return tokens
     };
 
+    // Parse the inner attributes `event_type` and `event_alias` and remove them from the macro
+    // outputs.
+    let (other_attrs, event_types): (Vec<_>, Vec<_>) = input
+        .attrs
+        .iter()
+        .cloned()
+        .partition(|attr| parse_event_type_attr(attr).is_none());
+    input.attrs = other_attrs;
+
     let subxt = utils::use_crate("substrate-subxt");
     let module = &input.ident;
     let module_name = module.to_string();
@@ -96,7 +142,7 @@ pub fn module(_args: TokenStream, tokens: TokenStream) -> TokenStream {
             None
         }
     });
-    let types = input.items.iter().filter_map(|item| {
+    let associated_types = input.items.iter().filter_map(|item| {
         if let syn::TraitItem::Type(ty) = item {
             if ignore(&ty.attrs) {
                 return None
@@ -110,6 +156,12 @@ pub fn module(_args: TokenStream, tokens: TokenStream) -> TokenStream {
             None
         }
     });
+    let types = event_types.iter().map(|attr| {
+        let (ident_str, ty) = parse_event_type_attr(&attr).unwrap();
+        quote! {
+            self.register_type_size::<#ty>(#ident_str);
+        }
+    });
 
     quote! {
         #input
@@ -127,6 +179,7 @@ pub fn module(_args: TokenStream, tokens: TokenStream) -> TokenStream {
         {
             fn #with_module(&mut self) {
                 #(#bounds)*
+                #(#associated_types)*
                 #(#types)*
             }
         }

proc-macro/src/utils.rs

@@ -214,6 +214,21 @@ impl<K: Parse, V: Parse> Parse for Attr<K, V> {
     }
 }
 
+#[derive(Debug)]
+pub struct UniAttr<A> {
+    pub paren: syn::token::Paren,
+    pub attr: A,
+}
+
+impl<A: Parse> Parse for UniAttr<A> {
+    fn parse(input: ParseStream) -> syn::Result<Self> {
+        let content;
+        let paren = syn::parenthesized!(content in input);
+        let attr = content.parse()?;
+        Ok(Self { paren, attr })
+    }
+}
+
 #[cfg(test)]
 pub(crate) fn assert_proc_macro(
     result: proc_macro2::TokenStream,

src/events.rs

@@ -32,6 +32,7 @@ use std::{
         HashMap,
         HashSet,
     },
+    fmt,
     marker::{
         PhantomData,
         Send,
@@ -71,20 +72,48 @@ impl std::fmt::Debug for RawEvent {
     }
 }
 
+trait TypeSegmenter: Send {
+    /// Consumes an object from an input stream, and output the serialized bytes.
+    fn segment(&self, input: &mut &[u8], output: &mut Vec<u8>) -> Result<(), Error>;
+}
+
+#[derive(Default)]
+struct TypeMarker<T>(PhantomData<T>);
+impl<T> TypeSegmenter for TypeMarker<T>
+where
+    T: Codec + Send,
+{
+    fn segment(&self, input: &mut &[u8], output: &mut Vec<u8>) -> Result<(), Error> {
+        T::decode(input).map_err(Error::from)?.encode_to(output);
+        Ok(())
+    }
+}
+
 /// Events decoder.
-#[derive(Debug)]
 pub struct EventsDecoder<T> {
     metadata: Metadata,
-    type_sizes: HashMap<String, usize>,
+    type_segmenters: HashMap<String, Box<dyn TypeSegmenter>>,
     marker: PhantomData<fn() -> T>,
 }
 
+impl<T> fmt::Debug for EventsDecoder<T> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("EventsDecoder<T>")
+            .field("metadata", &self.metadata)
+            .field(
+                "type_segmenters",
+                &self.type_segmenters.keys().cloned().collect::<String>(),
+            )
+            .finish()
+    }
+}
+
 impl<T: System> EventsDecoder<T> {
     /// Creates a new `EventsDecoder`.
     pub fn new(metadata: Metadata) -> Self {
         let mut decoder = Self {
             metadata,
-            type_sizes: HashMap::new(),
+            type_segmenters: HashMap::new(),
             marker: PhantomData,
         };
         // register default event arg type sizes for dynamic decoding of events
@@ -109,6 +138,8 @@ impl<T: System> EventsDecoder<T> {
         decoder.register_type_size::<T::BlockNumber>("BlockNumber");
         decoder.register_type_size::<T::Hash>("Hash");
         decoder.register_type_size::<u8>("VoteThreshold");
+        // Additional types
+        decoder.register_type_size::<(T::BlockNumber, u32)>("TaskAddress<BlockNumber>");
         decoder
     }
 
@@ -118,7 +149,10 @@ impl<T: System> EventsDecoder<T> {
         U: Default + Codec + Send + 'static,
     {
         let size = U::default().encode().len();
-        self.type_sizes.insert(name.to_string(), size);
+        // A segmenter decodes a type from an input stream (&mut &[u8]) and returns the serialized
+        // type to the output stream (&mut Vec<u8>).
+        self.type_segmenters
+            .insert(name.to_string(), Box::new(TypeMarker::<U>::default()));
         size
     }
 
@@ -129,7 +163,7 @@ impl<T: System> EventsDecoder<T> {
             for event in module.events() {
                 for arg in event.arguments() {
                     for primitive in arg.primitives() {
-                        if !self.type_sizes.contains_key(&primitive) {
+                        if !self.type_segmenters.contains_key(&primitive) {
                             missing.insert(format!(
                                 "{}::{}::{}",
                                 module.name(),
@@ -150,10 +184,10 @@ impl<T: System> EventsDecoder<T> {
         }
     }
 
-    fn decode_raw_bytes<I: Input, W: Output>(
+    fn decode_raw_bytes<W: Output>(
         &self,
         args: &[EventArg],
-        input: &mut I,
+        input: &mut &[u8],
         output: &mut W,
         errors: &mut Vec<RuntimeError>,
     ) -> Result<(), Error> {
@@ -188,9 +222,9 @@ impl<T: System> EventsDecoder<T> {
                         "DispatchResult" => DispatchResult::decode(input)?,
                         "DispatchError" => Err(DispatchError::decode(input)?),
                         _ => {
-                            if let Some(size) = self.type_sizes.get(name) {
-                                let mut buf = vec![0; *size];
-                                input.read(&mut buf)?;
+                            if let Some(seg) = self.type_segmenters.get(name) {
+                                let mut buf = Vec::<u8>::new();
+                                seg.segment(input, &mut buf)?;
                                 output.write(&buf);
                                 Ok(())
                             } else {
@@ -268,9 +302,12 @@ impl<T: System> EventsDecoder<T> {
     }
 }
 
+/// Raw event or error event
 #[derive(Debug)]
 pub enum Raw {
+    /// Event
     Event(RawEvent),
+    /// Error
     Error(RuntimeError),
 }
 

src/frame/session.rs

@@ -15,9 +15,15 @@
 // along with substrate-subxt.  If not, see <http://www.gnu.org/licenses/>.
 
 //! Session support
-use crate::frame::system::{
-    System,
-    SystemEventsDecoder as _,
+use crate::frame::{
+    balances::{
+        Balances,
+        BalancesEventsDecoder as _,
+    },
+    system::{
+        System,
+        SystemEventsDecoder as _,
+    },
 };
 use codec::Encode;
 use frame_support::Parameter;
@@ -45,9 +51,18 @@ macro_rules! default_impl {
     };
 }
 
+type IdentificationTuple<T> = (
+    <T as Session>::ValidatorId,
+    pallet_staking::Exposure<<T as System>::AccountId, <T as Balances>::Balance>,
+);
+
 /// The trait needed for this module.
 #[module]
-pub trait Session: System {
+pub trait Session: System + Balances {
+    #![event_alias(IdentificationTuple = IdentificationTuple<T>)]
+    #![event_alias(OpaqueTimeSlot = Vec<u8>)]
+    #![event_alias(SessionIndex = u32)]
+
     /// The validator account identifier type for the runtime.
     type ValidatorId: Parameter + Debug + Ord + Default + Send + Sync + 'static;
 

src/frame/staking.rs

@@ -62,9 +62,21 @@ pub struct SetPayeeCall<T: Staking> {
     pub _runtime: PhantomData<T>,
 }
 
+/// Identity of a Grandpa authority.
+pub type AuthorityId = crate::runtimes::app::grandpa::Public;
+/// The weight of an authority.
+pub type AuthorityWeight = u64;
+/// A list of Grandpa authorities with associated weights.
+pub type AuthorityList = Vec<(AuthorityId, AuthorityWeight)>;
+
 /// The subset of the `frame::Trait` that a client must implement.
 #[module]
-pub trait Staking: Balances {}
+#[rustfmt::skip]
+pub trait Staking: Balances {
+    #![event_alias(ElectionCompute = u8)]
+    #![event_type(EraIndex)]
+    #![event_type(AuthorityList)]
+}
 
 /// Number of eras to keep in history.
 ///