Metadata V16 (unstable): Enrich metadata with associated types of config traits (#5274)

This feature is part of the upcoming metadata V16. The associated types
of the `Config` trait that require the `TypeInfo` or `Parameter` bounds
are included in the metadata of the pallet. The metadata is not yet
exposed to the end-user, however the metadata intermediate
representation (IR) contains these types.

Developers can opt out of metadata collection of the associated types by
specifying `without_metadata` optional attribute to the
`#[pallet::config]`.

Furthermore, the `without_metadata` argument can be used in combination
with the newly added `#[pallet::include_metadata]` attribute to
selectively include only certain associated types in the metadata
collection.

### API Design

- There is nothing to collect from the associated types:

```rust
#[pallet::config]
pub trait Config: frame_system::Config {
		// Runtime events already propagated to the metadata.
		type RuntimeEvent: From<Event<Self>> + IsType<<Self as frame_system::Config>::RuntimeEvent>;

		// Constants are already propagated.
		#[pallet::constant]
		type MyGetParam2: Get<u32>;
	}
```

- Default automatic collection of associated types that require TypeInfo
or Parameter bounds:

```rust
	#[pallet::config]
	pub trait Config: frame_system::Config {
		// Runtime events already propagated to the metadata.
		type RuntimeEvent: From<Event<Self>> + IsType<<Self as frame_system::Config>::RuntimeEvent>;

		// Constants are already propagated.
		#[pallet::constant]
		type MyGetParam2: Get<u32>;

		// Associated type included by default, because it requires TypeInfo bound.
		/// Nonce doc.
		type Nonce: TypeInfo;

		// Associated type included by default, because it requires
		// Parameter bound (indirect TypeInfo).
		type AccountData: Parameter;

		// Associated type without metadata bounds, not included.
		type NotIncluded: From<u8>;
	}
```

- Disable automatic collection

```rust
// Associated types are not collected by default.
	#[pallet::config(without_metadata)]
	pub trait Config: frame_system::Config {
		// Runtime events already propagated to the metadata.
		type RuntimeEvent: From<Event<Self>> + IsType<<Self as frame_system::Config>::RuntimeEvent>;

		// Constants are already propagated.
		#[pallet::constant]
		type MyGetParam2: Get<u32>;

		// Explicitly include associated types.
		#[pallet::include_metadata]
		type Nonce: TypeInfo;

		type AccountData: Parameter;

		type NotIncluded: From<u8>;
	}
```

Builds on top of the PoC:
#4358
Closes: #4519

cc @paritytech/subxt-team

---------

Signed-off-by: Alexandru Vasile <[email protected]>
Co-authored-by: Bastian Köcher <[email protected]>
Co-authored-by: Guillaume Thiolliere <[email protected]>
Co-authored-by: Shawn Tabrizi <[email protected]>

Author: 4 people

prdoc/pr_5274.prdoc

@@ -0,0 +1,24 @@
+title: Enrich metadata IR with associated types of config traits
+
+doc:
+  - audience: Runtime Dev
+    description: |
+      This feature is part of the upcoming metadata V16. The associated types of the `Config` trait that require the `TypeInfo`
+      or `Parameter` bounds are included in the metadata of the pallet. The metadata is not yet exposed to the end-user, however
+      the metadata intermediate representation (IR) contains these types.
+
+      Developers can opt out of metadata collection of the associated types by specifying `without_metadata` optional attribute
+      to the `#[pallet::config]`.
+
+      Furthermore, the `without_metadata` argument can be used in combination with the newly added `#[pallet::include_metadata]`
+      attribute to selectively include only certain associated types in the metadata collection.
+
+crates:
+  - name: frame-support
+    bump: patch
+  - name: frame-support-procedural
+    bump: patch
+  - name: frame-support-procedural-tools
+    bump: patch
+  - name: sp-metadata-ir
+    bump: major

substrate/frame/support/procedural/src/construct_runtime/expand/metadata.rs

@@ -49,6 +49,7 @@ pub fn expand_runtime_metadata(
 			let event = expand_pallet_metadata_events(&filtered_names, runtime, decl);
 			let constants = expand_pallet_metadata_constants(runtime, decl);
 			let errors = expand_pallet_metadata_errors(runtime, decl);
+			let associated_types = expand_pallet_metadata_associated_types(runtime, decl);
 			let docs = expand_pallet_metadata_docs(runtime, decl);
 			let attr = decl.cfg_pattern.iter().fold(TokenStream::new(), |acc, pattern| {
 				let attr = TokenStream::from_str(&format!("#[cfg({})]", pattern.original()))
@@ -70,6 +71,7 @@ pub fn expand_runtime_metadata(
 					constants: #constants,
 					error: #errors,
 					docs: #docs,
+					associated_types: #associated_types,
 					deprecation_info: #deprecation_info,
 				}
 			}
@@ -261,3 +263,12 @@ fn expand_pallet_metadata_docs(runtime: &Ident, decl: &Pallet) -> TokenStream {
 		#path::Pallet::<#runtime #(, #path::#instance)*>::pallet_documentation_metadata()
 	}
 }
+
+fn expand_pallet_metadata_associated_types(runtime: &Ident, decl: &Pallet) -> TokenStream {
+	let path = &decl.path;
+	let instance = decl.instance.as_ref().into_iter();
+
+	quote! {
+		#path::Pallet::<#runtime #(, #path::#instance)*>::pallet_associated_types_metadata()
+	}
+}

substrate/frame/support/procedural/src/lib.rs

@@ -972,6 +972,15 @@ pub fn event(_: TokenStream, _: TokenStream) -> TokenStream {
 	pallet_macro_stub()
 }
 
+///
+/// ---
+///
+/// Documentation for this macro can be found at `frame_support::pallet_macros::include_metadata`.
+#[proc_macro_attribute]
+pub fn include_metadata(_: TokenStream, _: TokenStream) -> TokenStream {
+	pallet_macro_stub()
+}
+
 ///
 /// ---
 ///

substrate/frame/support/procedural/src/pallet/expand/config.rs

@@ -95,3 +95,51 @@ Consequently, a runtime that wants to include this pallet must implement this tr
 		_ => Default::default(),
 	}
 }
+
+/// Generate the metadata for the associated types of the config trait.
+///
+/// Implements the `pallet_associated_types_metadata` function for the pallet.
+pub fn expand_config_metadata(def: &Def) -> proc_macro2::TokenStream {
+	let frame_support = &def.frame_support;
+	let type_impl_gen = &def.type_impl_generics(proc_macro2::Span::call_site());
+	let type_use_gen = &def.type_use_generics(proc_macro2::Span::call_site());
+	let pallet_ident = &def.pallet_struct.pallet;
+	let trait_use_gen = &def.trait_use_generics(proc_macro2::Span::call_site());
+
+	let mut where_clauses = vec![&def.config.where_clause];
+	where_clauses.extend(def.extra_constants.iter().map(|d| &d.where_clause));
+	let completed_where_clause = super::merge_where_clauses(&where_clauses);
+
+	let types = def.config.associated_types_metadata.iter().map(|metadata| {
+		let ident = &metadata.ident;
+		let span = ident.span();
+		let ident_str = ident.to_string();
+		let cfgs = &metadata.cfg;
+
+		let no_docs = vec![];
+		let doc = if cfg!(feature = "no-metadata-docs") { &no_docs } else { &metadata.doc };
+
+		quote::quote_spanned!(span => {
+			#( #cfgs ) *
+			#frame_support::__private::metadata_ir::PalletAssociatedTypeMetadataIR {
+				name: #ident_str,
+				ty: #frame_support::__private::scale_info::meta_type::<
+						<T as Config #trait_use_gen>::#ident
+					>(),
+				docs: #frame_support::__private::sp_std::vec![ #( #doc ),* ],
+			}
+		})
+	});
+
+	quote::quote!(
+		impl<#type_impl_gen> #pallet_ident<#type_use_gen> #completed_where_clause {
+
+			#[doc(hidden)]
+			pub fn pallet_associated_types_metadata()
+				-> #frame_support::__private::sp_std::vec::Vec<#frame_support::__private::metadata_ir::PalletAssociatedTypeMetadataIR>
+			{
+				#frame_support::__private::sp_std::vec![ #( #types ),* ]
+			}
+		}
+	)
+}

substrate/frame/support/procedural/src/pallet/expand/mod.rs

@@ -60,6 +60,7 @@ pub fn expand(mut def: Def) -> proc_macro2::TokenStream {
 	let constants = constants::expand_constants(&mut def);
 	let pallet_struct = pallet_struct::expand_pallet_struct(&mut def);
 	let config = config::expand_config(&mut def);
+	let associated_types = config::expand_config_metadata(&def);
 	let call = call::expand_call(&mut def);
 	let tasks = tasks::expand_tasks(&mut def);
 	let error = error::expand_error(&mut def);
@@ -101,6 +102,7 @@ storage item. Otherwise, all storage items are listed among [*Type Definitions*]
 		#constants
 		#pallet_struct
 		#config
+		#associated_types
 		#call
 		#tasks
 		#error

substrate/frame/support/procedural/src/pallet/parse/config.rs

@@ -16,9 +16,9 @@
 // limitations under the License.
 
 use super::helper;
-use frame_support_procedural_tools::{get_doc_literals, is_using_frame_crate};
+use frame_support_procedural_tools::{get_cfg_attributes, get_doc_literals, is_using_frame_crate};
 use quote::ToTokens;
-use syn::{spanned::Spanned, token, Token};
+use syn::{spanned::Spanned, token, Token, TraitItemType};
 
 /// List of additional token to be used for parsing.
 mod keyword {
@@ -36,6 +36,7 @@ mod keyword {
 	syn::custom_keyword!(no_default);
 	syn::custom_keyword!(no_default_bounds);
 	syn::custom_keyword!(constant);
+	syn::custom_keyword!(include_metadata);
 }
 
 #[derive(Default)]
@@ -55,6 +56,8 @@ pub struct ConfigDef {
 	pub has_instance: bool,
 	/// Const associated type.
 	pub consts_metadata: Vec<ConstMetadataDef>,
+	/// Associated types metadata.
+	pub associated_types_metadata: Vec<AssociatedTypeMetadataDef>,
 	/// Whether the trait has the associated type `Event`, note that those bounds are
 	/// checked:
 	/// * `IsType<Self as frame_system::Config>::RuntimeEvent`
@@ -70,6 +73,26 @@ pub struct ConfigDef {
 	pub default_sub_trait: Option<DefaultTrait>,
 }
 
+/// Input definition for an associated type in pallet config.
+pub struct AssociatedTypeMetadataDef {
+	/// Name of the associated type.
+	pub ident: syn::Ident,
+	/// The doc associated.
+	pub doc: Vec<syn::Expr>,
+	/// The cfg associated.
+	pub cfg: Vec<syn::Attribute>,
+}
+
+impl From<&syn::TraitItemType> for AssociatedTypeMetadataDef {
+	fn from(trait_ty: &syn::TraitItemType) -> Self {
+		let ident = trait_ty.ident.clone();
+		let doc = get_doc_literals(&trait_ty.attrs);
+		let cfg = get_cfg_attributes(&trait_ty.attrs);
+
+		Self { ident, doc, cfg }
+	}
+}
+
 /// Input definition for a constant in pallet config.
 pub struct ConstMetadataDef {
 	/// Name of the associated type.
@@ -146,6 +169,8 @@ pub enum PalletAttrType {
 	NoBounds(keyword::no_default_bounds),
 	#[peek(keyword::constant, name = "constant")]
 	Constant(keyword::constant),
+	#[peek(keyword::include_metadata, name = "include_metadata")]
+	IncludeMetadata(keyword::include_metadata),
 }
 
 /// Parsing for `#[pallet::X]`
@@ -322,12 +347,32 @@ pub fn replace_self_by_t(input: proc_macro2::TokenStream) -> proc_macro2::TokenS
 		.collect()
 }
 
+/// Check that the trait item requires the `TypeInfo` bound (or similar).
+fn contains_type_info_bound(ty: &TraitItemType) -> bool {
+	const KNOWN_TYPE_INFO_BOUNDS: &[&str] = &[
+		// Explicit TypeInfo trait.
+		"TypeInfo",
+		// Implicit known substrate traits that implement type info.
+		// Note: Aim to keep this list as small as possible.
+		"Parameter",
+	];
+
+	ty.bounds.iter().any(|bound| {
+		let syn::TypeParamBound::Trait(bound) = bound else { return false };
+
+		KNOWN_TYPE_INFO_BOUNDS
+			.iter()
+			.any(|known| bound.path.segments.last().map_or(false, |last| last.ident == *known))
+	})
+}
+
 impl ConfigDef {
 	pub fn try_from(
 		frame_system: &syn::Path,
 		index: usize,
 		item: &mut syn::Item,
 		enable_default: bool,
+		disable_associated_metadata: bool,
 	) -> syn::Result<Self> {
 		let syn::Item::Trait(item) = item else {
 			let msg = "Invalid pallet::config, expected trait definition";
@@ -368,6 +413,7 @@ impl ConfigDef {
 
 		let mut has_event_type = false;
 		let mut consts_metadata = vec![];
+		let mut associated_types_metadata = vec![];
 		let mut default_sub_trait = if enable_default {
 			Some(DefaultTrait {
 				items: Default::default(),
@@ -383,6 +429,7 @@ impl ConfigDef {
 			let mut already_no_default = false;
 			let mut already_constant = false;
 			let mut already_no_default_bounds = false;
+			let mut already_collected_associated_type = None;
 
 			while let Ok(Some(pallet_attr)) =
 				helper::take_first_item_pallet_attr::<PalletAttr>(trait_item)
@@ -403,11 +450,29 @@ impl ConfigDef {
 							trait_item.span(),
 							"Invalid #[pallet::constant] in #[pallet::config], expected type item",
 						)),
+					// Pallet developer has explicitly requested to include metadata for this associated type.
+					//
+					// They must provide a type item that implements `TypeInfo`.
+					(PalletAttrType::IncludeMetadata(_), syn::TraitItem::Type(ref typ)) => {
+						if already_collected_associated_type.is_some() {
+							return Err(syn::Error::new(
+								pallet_attr._bracket.span.join(),
+								"Duplicate #[pallet::include_metadata] attribute not allowed.",
+							));
+						}
+						already_collected_associated_type = Some(pallet_attr._bracket.span.join());
+						associated_types_metadata.push(AssociatedTypeMetadataDef::from(AssociatedTypeMetadataDef::from(typ)));
+					}
+					(PalletAttrType::IncludeMetadata(_), _) =>
+						return Err(syn::Error::new(
+							pallet_attr._bracket.span.join(),
+							"Invalid #[pallet::include_metadata] in #[pallet::config], expected type item",
+						)),
 					(PalletAttrType::NoDefault(_), _) => {
 						if !enable_default {
 							return Err(syn::Error::new(
 								pallet_attr._bracket.span.join(),
-								"`#[pallet:no_default]` can only be used if `#[pallet::config(with_default)]` \
+								"`#[pallet::no_default]` can only be used if `#[pallet::config(with_default)]` \
 								has been specified"
 							));
 						}
@@ -439,6 +504,47 @@ impl ConfigDef {
 				}
 			}
 
+			if let Some(span) = already_collected_associated_type {
+				// Events and constants are already propagated to the metadata
+				if is_event {
+					return Err(syn::Error::new(
+						span,
+						"Invalid #[pallet::include_metadata] for `type RuntimeEvent`. \
+						The associated type `RuntimeEvent` is already collected in the metadata.",
+					))
+				}
+
+				if already_constant {
+					return Err(syn::Error::new(
+						span,
+						"Invalid #[pallet::include_metadata]: conflict with #[pallet::constant]. \
+						Pallet constant already collect the metadata for the type.",
+					))
+				}
+
+				if let syn::TraitItem::Type(ref ty) = trait_item {
+					if !contains_type_info_bound(ty) {
+						let msg = format!(
+						"Invalid #[pallet::include_metadata] in #[pallet::config], collected type `{}` \
+						does not implement `TypeInfo` or `Parameter`",
+						ty.ident,
+					);
+						return Err(syn::Error::new(span, msg));
+					}
+				}
+			} else {
+				// Metadata of associated types is collected by default, if the associated type
+				// implements `TypeInfo`, or a similar trait that requires the `TypeInfo` bound.
+				if !disable_associated_metadata && !is_event && !already_constant {
+					if let syn::TraitItem::Type(ref ty) = trait_item {
+						// Collect the metadata of the associated type if it implements `TypeInfo`.
+						if contains_type_info_bound(ty) {
+							associated_types_metadata.push(AssociatedTypeMetadataDef::from(ty));
+						}
+					}
+				}
+			}
+
 			if !already_no_default && enable_default {
 				default_sub_trait
 					.as_mut()
@@ -481,6 +587,7 @@ impl ConfigDef {
 			index,
 			has_instance,
 			consts_metadata,
+			associated_types_metadata,
 			has_event_type,
 			where_clause,
 			default_sub_trait,