Add support for async/streams/futures to Rust generator

This adds support for generating bindings which use the [Async
ABI](https://github.com/WebAssembly/component-model/blob/main/design/mvp/Async.md)
along with the [`stream`, `future`, and
`error-context`](WebAssembly/component-model#405) types.

By default, normal synchronous bindings are generated, but the user may opt-in
to async bindings for all or some of the imported and/or exported functions in
the target world and interfaces -- provided the default-enabled `async` feature
is enabled.

In addition, we generate `StreamPayload` and/or `FuturePayload` trait
implementations for any types appearing as the `T` in `stream<T>` or `future<T>`
in the WIT files, respectively.  That enables user code to call `new_stream` or
`new_future` to create `stream`s or `future`s with those payload types, then
write to them, read from them, and/or pass the readable end as a parameter to a
component import or return value of a component export.

Note that I've added new `core::abi::Instruction` enum variants to handle async
lifting and lowering, but they're currently tailored to the Rust generator and
will probably change somewhat as we add support for other languages.

This does not include any new tests; I'll add those in a follow-up commit.

Signed-off-by: Joel Dice <[email protected]>

add `async: true` case to Rust `codegen_tests`

This ensures that all the codegen test WIT files produce compile-able bindings
with `async: true` (i.e. all imports lowered and all exports lifted using the
async ABI).  That revealed some issues involving resource methods and
constructors, as well as missing stub support, which I've resolved.

Signed-off-by: Joel Dice <[email protected]>

add codegen tests for futures, streams, and error-contexts

Signed-off-by: Joel Dice <[email protected]>

Author: dicej

Cargo.toml

@@ -31,12 +31,13 @@ clap = { version = "4.3.19", features = ["derive"] }
 indexmap = "2.0.0"
 prettyplease = "0.2.20"
 syn = { version = "2.0", features = ["printing"] }
+futures = "0.3.31"
 
-wasmparser = "0.220.0"
-wasm-encoder = "0.220.0"
-wasm-metadata = "0.220.0"
-wit-parser = "0.220.0"
-wit-component = "0.220.0"
+wasmparser = { git = "https://github.com/dicej/wasm-tools", branch = "async" }
+wasm-encoder = { git = "https://github.com/dicej/wasm-tools", branch = "async" }
+wasm-metadata = { git = "https://github.com/dicej/wasm-tools", branch = "async" }
+wit-parser = { git = "https://github.com/dicej/wasm-tools", branch = "async" }
+wit-component = { git = "https://github.com/dicej/wasm-tools", branch = "async" }
 
 wit-bindgen-core = { path = 'crates/core', version = '0.35.0' }
 wit-bindgen-c = { path = 'crates/c', version = '0.35.0' }
@@ -74,6 +75,7 @@ default = [
   'go',
   'csharp',
   'moonbit',
+  'async',
 ]
 c = ['dep:wit-bindgen-c']
 rust = ['dep:wit-bindgen-rust']
@@ -83,6 +85,7 @@ go = ['dep:wit-bindgen-go']
 csharp = ['dep:wit-bindgen-csharp']
 csharp-mono = ['csharp']
 moonbit = ['dep:wit-bindgen-moonbit']
+async = ["wit-bindgen-rust/async"]
 
 [dev-dependencies]
 heck = { workspace = true }

crates/c/src/lib.rs

@@ -718,6 +718,7 @@ fn is_prim_type_id(resolve: &Resolve, id: TypeId) -> bool {
         | TypeDefKind::Result(_)
         | TypeDefKind::Future(_)
         | TypeDefKind::Stream(_)
+        | TypeDefKind::ErrorContext
         | TypeDefKind::Unknown => false,
     }
 }
@@ -779,8 +780,9 @@ pub fn push_ty_name(resolve: &Resolve, ty: &Type, src: &mut String) {
                     src.push_str("list_");
                     push_ty_name(resolve, ty, src);
                 }
-                TypeDefKind::Future(_) => unimplemented!(),
-                TypeDefKind::Stream(_) => unimplemented!(),
+                TypeDefKind::Future(_) => todo!(),
+                TypeDefKind::Stream(_) => todo!(),
+                TypeDefKind::ErrorContext => todo!(),
                 TypeDefKind::Handle(Handle::Own(resource)) => {
                     src.push_str("own_");
                     push_ty_name(resolve, &Type::Id(*resource), src);
@@ -992,6 +994,7 @@ impl Return {
 
             TypeDefKind::Future(_) => todo!("return_single for future"),
             TypeDefKind::Stream(_) => todo!("return_single for stream"),
+            TypeDefKind::ErrorContext => todo!("return_single for error-context"),
             TypeDefKind::Resource => todo!("return_single for resource"),
             TypeDefKind::Unknown => unreachable!(),
         }
@@ -1339,6 +1342,21 @@ void __wasm_export_{ns}_{snake}_dtor({ns}_{snake}_t* arg) {{
         self.finish_typedef_struct(id);
     }
 
+    fn type_future(&mut self, id: TypeId, name: &str, ty: &Option<Type>, docs: &Docs) {
+        _ = (id, name, ty, docs);
+        todo!()
+    }
+
+    fn type_stream(&mut self, id: TypeId, name: &str, ty: &Type, docs: &Docs) {
+        _ = (id, name, ty, docs);
+        todo!()
+    }
+
+    fn type_error_context(&mut self, id: TypeId, name: &str, docs: &Docs) {
+        _ = (id, name, docs);
+        todo!()
+    }
+
     fn type_builtin(&mut self, id: TypeId, name: &str, ty: &Type, docs: &Docs) {
         let _ = (id, name, ty, docs);
     }
@@ -1427,12 +1445,16 @@ impl<'a> wit_bindgen_core::AnonymousTypeGenerator<'a> for InterfaceGenerator<'a>
         todo!("print_anonymous_type for future");
     }
 
-    fn anonymous_type_stream(&mut self, _id: TypeId, _ty: &Stream, _docs: &Docs) {
+    fn anonymous_type_stream(&mut self, _id: TypeId, _ty: &Type, _docs: &Docs) {
         todo!("print_anonymous_type for stream");
     }
 
-    fn anonymous_typ_type(&mut self, _id: TypeId, _ty: &Type, _docs: &Docs) {
-        todo!("print_anonymous_type for typ");
+    fn anonymous_type_error_context(&mut self) {
+        todo!("print_anonymous_type for error-context");
+    }
+
+    fn anonymous_type_type(&mut self, _id: TypeId, _ty: &Type, _docs: &Docs) {
+        todo!("print_anonymous_type for type");
     }
 }
 
@@ -1605,6 +1627,7 @@ impl InterfaceGenerator<'_> {
             }
             TypeDefKind::Future(_) => todo!("print_dtor for future"),
             TypeDefKind::Stream(_) => todo!("print_dtor for stream"),
+            TypeDefKind::ErrorContext => todo!("print_dtor for error-context"),
             TypeDefKind::Resource => {}
             TypeDefKind::Handle(Handle::Borrow(id) | Handle::Own(id)) => {
                 self.free(&Type::Id(*id), "*ptr");
@@ -1750,6 +1773,7 @@ impl InterfaceGenerator<'_> {
             LiftLower::LowerArgsLiftResults,
             func,
             &mut f,
+            false,
         );
 
         let FunctionBindgen {
@@ -1822,6 +1846,7 @@ impl InterfaceGenerator<'_> {
             LiftLower::LiftArgsLowerResults,
             func,
             &mut f,
+            false,
         );
         let FunctionBindgen { src, .. } = f;
         self.src.c_adapters(&src);
@@ -1852,7 +1877,7 @@ impl InterfaceGenerator<'_> {
 
             let mut f = FunctionBindgen::new(self, c_sig, &import_name);
             f.params = params;
-            abi::post_return(f.gen.resolve, func, &mut f);
+            abi::post_return(f.gen.resolve, func, &mut f, false);
             let FunctionBindgen { src, .. } = f;
             self.src.c_fns(&src);
             self.src.c_fns("}\n");
@@ -2075,17 +2100,8 @@ impl InterfaceGenerator<'_> {
 
                 TypeDefKind::List(ty) => self.contains_droppable_borrow(ty),
 
-                TypeDefKind::Future(r) => r
-                    .as_ref()
-                    .map_or(false, |ty| self.contains_droppable_borrow(ty)),
-
-                TypeDefKind::Stream(s) => {
-                    s.element
-                        .as_ref()
-                        .map_or(false, |ty| self.contains_droppable_borrow(ty))
-                        || s.end
-                            .as_ref()
-                            .map_or(false, |ty| self.contains_droppable_borrow(ty))
+                TypeDefKind::Future(_) | TypeDefKind::Stream(_) | TypeDefKind::ErrorContext => {
+                    false
                 }
 
                 TypeDefKind::Type(ty) => self.contains_droppable_borrow(ty),
@@ -2753,7 +2769,7 @@ impl Bindgen for FunctionBindgen<'_, '_> {
                 self.src.push_str(");\n");
             }
 
-            Instruction::CallInterface { func } => {
+            Instruction::CallInterface { func, .. } => {
                 let mut args = String::new();
                 for (i, (op, (byref, _))) in operands.iter().zip(&self.sig.params).enumerate() {
                     if i > 0 {
@@ -3037,6 +3053,10 @@ impl Bindgen for FunctionBindgen<'_, '_> {
                 uwriteln!(self.src, "}}");
             }
 
+            Instruction::Flush { amt } => {
+                results.extend(operands.iter().take(*amt).map(|v| v.clone()));
+            }
+
             i => unimplemented!("{:?}", i),
         }
     }
@@ -3145,6 +3165,7 @@ pub fn is_arg_by_pointer(resolve: &Resolve, ty: &Type) -> bool {
             TypeDefKind::Tuple(_) | TypeDefKind::Record(_) | TypeDefKind::List(_) => true,
             TypeDefKind::Future(_) => todo!("is_arg_by_pointer for future"),
             TypeDefKind::Stream(_) => todo!("is_arg_by_pointer for stream"),
+            TypeDefKind::ErrorContext => todo!("is_arg_by_pointer for error-context"),
             TypeDefKind::Resource => todo!("is_arg_by_pointer for resource"),
             TypeDefKind::Unknown => unreachable!(),
         },

crates/c/tests/codegen.rs

@@ -6,6 +6,14 @@ use std::process::Command;
 use wit_parser::{Resolve, UnresolvedPackageGroup};
 
 macro_rules! codegen_test {
+    // TODO: implement support for stream, future, and error-context, and then
+    // remove these lines:
+    (streams $name:tt $test:tt) => {};
+    (futures $name:tt $test:tt) => {};
+    (resources_with_streams $name:tt $test:tt) => {};
+    (resources_with_futures $name:tt $test:tt) => {};
+    (error_context $name:tt $test:tt) => {};
+
     ($id:ident $name:tt $test:tt) => {
         #[test]
         fn $id() {

crates/core/Cargo.toml

@@ -18,3 +18,7 @@ doctest = false
 wit-parser = { workspace = true }
 anyhow = { workspace = true }
 heck = { workspace = true }
+
+[features]
+default = ["async"]
+async = []