From 57b1acee689e49f43338e4efb15b65e9f922831d Mon Sep 17 00:00:00 2001 From: Joel Dice Date: Thu, 19 Dec 2024 15:22:24 -0700 Subject: [PATCH] add async support to `wit_component::dummy_module` (#1960) This allows us to round-trip fuzz test using the async ABI(s) as well as the sync one. I've also added corresponding `--async-callback` and `--async-stackful` options to the `component embed --dummy` subcommand for generating dummy modules which use the new ABIs. Note that this currently only generates ultra-minimal, non-functional modules. A real module would import the `task.return` intrinsic with the appropriate signature for each async export, and would presumably use other new intrinsics such as `subtask.drop`, `task.backpressure`, etc. -- not to mention the various `stream.*`, `future.*`, and `error-context.*` intrinsics. For more thorough fuzz testing, we'll want to generate imports for all known intrinsics (although we probably wouldn't do that for `component embed --dummy` modules, since it would be more confusing than helpful. Signed-off-by: Joel Dice --- crates/wit-component/src/dummy.rs | 60 ++++++++++++------ crates/wit-component/src/encoding.rs | 4 +- crates/wit-component/src/semver_check.rs | 4 +- crates/wit-parser/src/lib.rs | 81 ++++++++++++++++++++++++ crates/wit-parser/src/resolve.rs | 74 +++++++++++++++------- fuzz/src/roundtrip_wit.rs | 14 ++-- src/bin/wasm-tools/component.rs | 44 ++++++++++++- 7 files changed, 225 insertions(+), 56 deletions(-) diff --git a/crates/wit-component/src/dummy.rs b/crates/wit-component/src/dummy.rs index 78980e73d1..03ce835fc4 100644 --- a/crates/wit-component/src/dummy.rs +++ b/crates/wit-component/src/dummy.rs @@ -1,18 +1,18 @@ -use wit_parser::abi::{AbiVariant, WasmType}; +use wit_parser::abi::WasmType; use wit_parser::{ - Function, Mangling, Resolve, ResourceIntrinsic, TypeDefKind, TypeId, WasmExport, WasmImport, - WorldId, WorldItem, WorldKey, + Function, LiftLowerAbi, ManglingAndAbi, Resolve, ResourceIntrinsic, TypeDefKind, TypeId, + WasmExport, WasmExportKind, WasmImport, WorldId, WorldItem, WorldKey, }; /// Generate a dummy implementation core Wasm module for a given WIT document -pub fn dummy_module(resolve: &Resolve, world: WorldId, mangling: Mangling) -> Vec { +pub fn dummy_module(resolve: &Resolve, world: WorldId, mangling: ManglingAndAbi) -> Vec { let world = &resolve.worlds[world]; let mut wat = String::new(); wat.push_str("(module\n"); for (name, import) in world.imports.iter() { match import { WorldItem::Function(func) => { - let sig = resolve.wasm_signature(AbiVariant::GuestImport, func); + let sig = resolve.wasm_signature(mangling.import_variant(), func); let (module, name) = resolve.wasm_import_name( mangling, @@ -29,7 +29,7 @@ pub fn dummy_module(resolve: &Resolve, world: WorldId, mangling: Mangling) -> Ve } WorldItem::Interface { id: import, .. } => { for (_, func) in resolve.interfaces[*import].functions.iter() { - let sig = resolve.wasm_signature(AbiVariant::GuestImport, func); + let sig = resolve.wasm_signature(mangling.import_variant(), func); let (module, name) = resolve.wasm_import_name( mangling, @@ -139,7 +139,7 @@ pub fn dummy_module(resolve: &Resolve, world: WorldId, mangling: Mangling) -> Ve resolve: &Resolve, interface: Option<&WorldKey>, resource: TypeId, - mangling: Mangling, + mangling: ManglingAndAbi, ) { let ty = &resolve.types[resource]; match ty.kind { @@ -162,15 +162,15 @@ pub fn dummy_module(resolve: &Resolve, world: WorldId, mangling: Mangling) -> Ve resolve: &Resolve, interface: Option<&WorldKey>, func: &Function, - mangling: Mangling, + mangling: ManglingAndAbi, ) { - let sig = resolve.wasm_signature(AbiVariant::GuestExport, func); + let sig = resolve.wasm_signature(mangling.export_variant(), func); let name = resolve.wasm_export_name( mangling, WasmExport::Func { interface, func, - post_return: false, + kind: WasmExportKind::Normal, }, ); wat.push_str(&format!("(func (export \"{name}\")")); @@ -178,17 +178,35 @@ pub fn dummy_module(resolve: &Resolve, world: WorldId, mangling: Mangling) -> Ve push_tys(wat, "result", &sig.results); wat.push_str(" unreachable)\n"); - let name = resolve.wasm_export_name( - mangling, - WasmExport::Func { - interface, - func, - post_return: true, - }, - ); - wat.push_str(&format!("(func (export \"{name}\")")); - push_tys(wat, "param", &sig.results); - wat.push_str(")\n"); + match mangling { + ManglingAndAbi::Standard32 | ManglingAndAbi::Legacy(LiftLowerAbi::Sync) => { + let name = resolve.wasm_export_name( + mangling, + WasmExport::Func { + interface, + func, + kind: WasmExportKind::PostReturn, + }, + ); + wat.push_str(&format!("(func (export \"{name}\")")); + push_tys(wat, "param", &sig.results); + wat.push_str(")\n"); + } + ManglingAndAbi::Legacy(LiftLowerAbi::AsyncCallback) => { + let name = resolve.wasm_export_name( + mangling, + WasmExport::Func { + interface, + func, + kind: WasmExportKind::Callback, + }, + ); + wat.push_str(&format!( + "(func (export \"{name}\") (param i32 i32 i32 i32) (result i32) unreachable)\n" + )); + } + ManglingAndAbi::Legacy(LiftLowerAbi::AsyncStackful) => {} + } } fn push_tys(dst: &mut String, desc: &str, params: &[WasmType]) { diff --git a/crates/wit-component/src/encoding.rs b/crates/wit-component/src/encoding.rs index 14a12b9e82..6c1b3a25b8 100644 --- a/crates/wit-component/src/encoding.rs +++ b/crates/wit-component/src/encoding.rs @@ -2823,7 +2823,7 @@ impl ComponentWorld<'_> { mod test { use super::*; use crate::{dummy_module, embed_component_metadata}; - use wit_parser::Mangling; + use wit_parser::ManglingAndAbi; #[test] fn it_renames_imports() { @@ -2849,7 +2849,7 @@ world test { .unwrap(); let world = resolve.select_world(pkg, None).unwrap(); - let mut module = dummy_module(&resolve, world, Mangling::Standard32); + let mut module = dummy_module(&resolve, world, ManglingAndAbi::Standard32); embed_component_metadata(&mut module, &resolve, world, StringEncoding::UTF8).unwrap(); diff --git a/crates/wit-component/src/semver_check.rs b/crates/wit-component/src/semver_check.rs index 58d0aa74e2..5be11054fb 100644 --- a/crates/wit-component/src/semver_check.rs +++ b/crates/wit-component/src/semver_check.rs @@ -5,7 +5,7 @@ use crate::{ use anyhow::{bail, Context, Result}; use wasm_encoder::{ComponentBuilder, ComponentExportKind, ComponentTypeRef}; use wasmparser::Validator; -use wit_parser::{Mangling, Resolve, WorldId}; +use wit_parser::{ManglingAndAbi, Resolve, WorldId}; /// Tests whether `new` is a semver-compatible upgrade from the world `prev`. /// @@ -63,7 +63,7 @@ pub fn semver_check(mut resolve: Resolve, prev: WorldId, new: WorldId) -> Result let mut root_component = ComponentBuilder::default(); // (1) above - create a dummy component which has the shape of `prev`. - let mut prev_as_module = dummy_module(&resolve, prev, Mangling::Standard32); + let mut prev_as_module = dummy_module(&resolve, prev, ManglingAndAbi::Standard32); embed_component_metadata(&mut prev_as_module, &resolve, prev, StringEncoding::UTF8) .context("failed to embed component metadata")?; let prev_as_component = ComponentEncoder::default() diff --git a/crates/wit-parser/src/lib.rs b/crates/wit-parser/src/lib.rs index 09d38dc401..cb8b2f4ead 100644 --- a/crates/wit-parser/src/lib.rs +++ b/crates/wit-parser/src/lib.rs @@ -1,3 +1,4 @@ +use crate::abi::AbiVariant; use anyhow::{bail, Context, Result}; use id_arena::{Arena, Id}; use indexmap::IndexMap; @@ -939,6 +940,86 @@ impl std::str::FromStr for Mangling { } } +/// Possible lift/lower ABI choices supported when mangling names. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum LiftLowerAbi { + /// Both imports and exports will use the synchronous ABI. + Sync, + + /// Both imports and exports will use the async ABI (with a callback for + /// each export). + AsyncCallback, + + /// Both imports and exports will use the async ABI (with no callbacks for + /// exports). + AsyncStackful, +} + +impl LiftLowerAbi { + fn import_prefix(self) -> &'static str { + match self { + Self::Sync => "", + Self::AsyncCallback | Self::AsyncStackful => "[async]", + } + } + + /// Get the import [`AbiVariant`] corresponding to this [`LiftLowerAbi`] + pub fn import_variant(self) -> AbiVariant { + match self { + Self::Sync => AbiVariant::GuestImport, + Self::AsyncCallback | Self::AsyncStackful => AbiVariant::GuestImportAsync, + } + } + + fn export_prefix(self) -> &'static str { + match self { + Self::Sync => "", + Self::AsyncCallback => "[async]", + Self::AsyncStackful => "[async-stackful]", + } + } + + /// Get the export [`AbiVariant`] corresponding to this [`LiftLowerAbi`] + pub fn export_variant(self) -> AbiVariant { + match self { + Self::Sync => AbiVariant::GuestExport, + Self::AsyncCallback => AbiVariant::GuestExportAsync, + Self::AsyncStackful => AbiVariant::GuestExportAsyncStackful, + } + } +} + +/// Combination of [`Mangling`] and [`LiftLowerAbi`]. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum ManglingAndAbi { + /// See [`Mangling::Standard32`]. + /// + /// As of this writing, the standard name mangling only supports the + /// synchronous ABI. + Standard32, + + /// See [`Mangling::Legacy`] and [`LiftLowerAbi`]. + Legacy(LiftLowerAbi), +} + +impl ManglingAndAbi { + /// Get the import [`AbiVariant`] corresponding to this [`ManglingAndAbi`] + pub fn import_variant(self) -> AbiVariant { + match self { + Self::Standard32 => AbiVariant::GuestImport, + Self::Legacy(abi) => abi.import_variant(), + } + } + + /// Get the export [`AbiVariant`] corresponding to this [`ManglingAndAbi`] + pub fn export_variant(self) -> AbiVariant { + match self { + Self::Standard32 => AbiVariant::GuestExport, + Self::Legacy(abi) => abi.export_variant(), + } + } +} + impl Function { pub fn item_name(&self) -> &str { match &self.kind { diff --git a/crates/wit-parser/src/resolve.rs b/crates/wit-parser/src/resolve.rs index 93a82e3759..d1f40381ab 100644 --- a/crates/wit-parser/src/resolve.rs +++ b/crates/wit-parser/src/resolve.rs @@ -18,9 +18,9 @@ use crate::ast::{parse_use_path, ParsedUsePath}; use crate::serde_::{serialize_arena, serialize_id_map}; use crate::{ AstItem, Docs, Error, Function, FunctionKind, Handle, IncludeName, Interface, InterfaceId, - InterfaceSpan, Mangling, PackageName, PackageNotFoundError, Results, SourceMap, Stability, - Type, TypeDef, TypeDefKind, TypeId, TypeIdVisitor, TypeOwner, UnresolvedPackage, - UnresolvedPackageGroup, World, WorldId, WorldItem, WorldKey, WorldSpan, + InterfaceSpan, LiftLowerAbi, ManglingAndAbi, PackageName, PackageNotFoundError, Results, + SourceMap, Stability, Type, TypeDef, TypeDefKind, TypeId, TypeIdVisitor, TypeOwner, + UnresolvedPackage, UnresolvedPackageGroup, World, WorldId, WorldItem, WorldKey, WorldSpan, }; mod clone; @@ -2287,9 +2287,13 @@ package {name} is defined in two different locations:\n\ /// use `import` with the name `mangling` scheme specified as well. This can /// be useful for bindings generators, for example, and these names are /// recognized by `wit-component` and `wasm-tools component new`. - pub fn wasm_import_name(&self, mangling: Mangling, import: WasmImport<'_>) -> (String, String) { + pub fn wasm_import_name( + &self, + mangling: ManglingAndAbi, + import: WasmImport<'_>, + ) -> (String, String) { match mangling { - Mangling::Standard32 => match import { + ManglingAndAbi::Standard32 => match import { WasmImport::Func { interface, func } => { let module = match interface { Some(key) => format!("cm32p2|{}", self.name_canonicalized_world_key(key)), @@ -2321,13 +2325,13 @@ package {name} is defined in two different locations:\n\ (module, name) } }, - Mangling::Legacy => match import { + ManglingAndAbi::Legacy(abi) => match import { WasmImport::Func { interface, func } => { let module = match interface { Some(key) => self.name_world_key(key), None => format!("$root"), }; - (module, func.name.clone()) + (module, format!("{}{}", abi.import_prefix(), func.name)) } WasmImport::ResourceIntrinsic { interface, @@ -2354,22 +2358,22 @@ package {name} is defined in two different locations:\n\ format!("$root") } }; - (module, name) + (module, format!("{}{name}", abi.import_prefix())) } }, } } - /// Returns the core wasm export name for the specified `import`. + /// Returns the core wasm export name for the specified `export`. /// - /// This is the same as [`Resovle::wasm_import_name`], except for exports. - pub fn wasm_export_name(&self, mangling: Mangling, import: WasmExport<'_>) -> String { + /// This is the same as [`Resolve::wasm_import_name`], except for exports. + pub fn wasm_export_name(&self, mangling: ManglingAndAbi, export: WasmExport<'_>) -> String { match mangling { - Mangling::Standard32 => match import { + ManglingAndAbi::Standard32 => match export { WasmExport::Func { interface, func, - post_return, + kind, } => { let mut name = String::from("cm32p2|"); if let Some(interface) = interface { @@ -2378,8 +2382,13 @@ package {name} is defined in two different locations:\n\ } name.push_str("|"); name.push_str(&func.name); - if post_return { - name.push_str("_post"); + match kind { + WasmExportKind::Normal => {} + WasmExportKind::PostReturn => name.push_str("_post"), + WasmExportKind::Callback => todo!( + "not yet supported: \ + async callback functions using standard name mangling" + ), } name } @@ -2395,15 +2404,20 @@ package {name} is defined in two different locations:\n\ WasmExport::Initialize => "cm32p2_initialize".to_string(), WasmExport::Realloc => "cm32p2_realloc".to_string(), }, - Mangling::Legacy => match import { + ManglingAndAbi::Legacy(abi) => match export { WasmExport::Func { interface, func, - post_return, + kind, } => { - let mut name = String::new(); - if post_return { - name.push_str("cabi_post_"); + let mut name = abi.export_prefix().to_string(); + match kind { + WasmExportKind::Normal => {} + WasmExportKind::PostReturn => name.push_str("cabi_post_"), + WasmExportKind::Callback => { + assert!(matches!(abi, LiftLowerAbi::AsyncCallback)); + name = format!("[callback]{name}") + } } if let Some(interface) = interface { let s = self.name_world_key(interface); @@ -2419,7 +2433,7 @@ package {name} is defined in two different locations:\n\ } => { let name = self.types[resource].name.as_ref().unwrap(); let interface = self.name_world_key(interface); - format!("{interface}#[dtor]{name}") + format!("{}{interface}#[dtor]{name}", abi.export_prefix()) } WasmExport::Memory => "memory".to_string(), WasmExport::Initialize => "_initialize".to_string(), @@ -2467,6 +2481,20 @@ pub enum ResourceIntrinsic { ExportedRep, } +/// Indicates whether a function export is a normal export, a post-return +/// function, or a callback function. +#[derive(Debug)] +pub enum WasmExportKind { + /// Normal function export. + Normal, + + /// Post-return function. + PostReturn, + + /// Async callback function. + Callback, +} + /// Different kinds of exports that can be passed to /// [`Resolve::wasm_export_name`] to export from core wasm modules. #[derive(Debug)] @@ -2480,8 +2508,8 @@ pub enum WasmExport<'a> { /// The function being exported. func: &'a Function, - /// Whether or not this is a post-return function or not. - post_return: bool, + /// Kind of function (normal, post-return, or callback) being exported. + kind: WasmExportKind, }, /// A destructor for a resource exported from this module. diff --git a/fuzz/src/roundtrip_wit.rs b/fuzz/src/roundtrip_wit.rs index 79dbc6e2d9..504097dcc1 100644 --- a/fuzz/src/roundtrip_wit.rs +++ b/fuzz/src/roundtrip_wit.rs @@ -1,7 +1,7 @@ use arbitrary::{Result, Unstructured}; use std::path::Path; use wit_component::*; -use wit_parser::{Mangling, PackageId, Resolve}; +use wit_parser::{LiftLowerAbi, ManglingAndAbi, PackageId, Resolve}; pub fn run(u: &mut Unstructured<'_>) -> Result<()> { let wasm = u.arbitrary().and_then(|config| { @@ -36,9 +36,11 @@ pub fn run(u: &mut Unstructured<'_>) -> Result<()> { let mut decoded_bindgens = Vec::new(); for (id, world) in resolve.worlds.iter().take(20) { log::debug!("embedding world {} as in a dummy module", world.name); - let mangling = match u.int_in_range(0..=1)? { - 0 => Mangling::Legacy, - 1 => Mangling::Standard32, + let mangling = match u.int_in_range(0..=3)? { + 0 => ManglingAndAbi::Legacy(LiftLowerAbi::Sync), + 1 => ManglingAndAbi::Legacy(LiftLowerAbi::AsyncCallback), + 2 => ManglingAndAbi::Legacy(LiftLowerAbi::AsyncStackful), + 3 => ManglingAndAbi::Standard32, _ => unreachable!(), }; let mut dummy = wit_component::dummy_module(&resolve, id, mangling); @@ -53,7 +55,9 @@ pub fn run(u: &mut Unstructured<'_>) -> Result<()> { .encode() .unwrap(); write_file("dummy.component.wasm", &wasm); - wasmparser::Validator::new().validate_all(&wasm).unwrap(); + wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::all()) + .validate_all(&wasm) + .unwrap(); // Decode what was just created and record it later for testing merging // worlds together. diff --git a/src/bin/wasm-tools/component.rs b/src/bin/wasm-tools/component.rs index f62731b5fc..185493743c 100644 --- a/src/bin/wasm-tools/component.rs +++ b/src/bin/wasm-tools/component.rs @@ -16,7 +16,7 @@ use wit_component::{ embed_component_metadata, metadata, ComponentEncoder, DecodedWasm, Linker, StringEncoding, WitPrinter, }; -use wit_parser::{Mangling, PackageId, Resolve}; +use wit_parser::{LiftLowerAbi, Mangling, ManglingAndAbi, PackageId, Resolve}; /// WebAssembly wit-based component tooling. #[derive(Parser)] @@ -306,6 +306,26 @@ pub struct EmbedOpts { #[clap(long, conflicts_with = "dummy")] dummy_names: Option, + /// With `--dummy-names legacy`, this will generate a core module such that + /// all the imports are lowered using the async ABI and the exports are + /// lifted using the async-with-callback ABI. + /// + /// Note that this does not yet work with `--dummy` or `--dummy-names + /// standard32` because the standard name mangling scheme does not yet + /// support async-related features as of this writing. + #[clap(long, requires = "dummy_names", conflicts_with = "async_stackful")] + async_callback: bool, + + /// With `--dummy-names legacy`, this will generate a core module such that + /// all the imports are lowered using the async ABI and the exports are + /// lifted using the async-without-callback (i.e. stackful) ABI. + /// + /// Note that this does not yet work with `--dummy` or `--dummy-names + /// standard32` because the standard name mangling scheme does not yet + /// support async-related features as of this writing. + #[clap(long, requires = "dummy_names", conflicts_with = "async_callback")] + async_stackful: bool, + /// Print the output in the WebAssembly text format instead of binary. #[clap(long, short = 't')] wat: bool, @@ -344,9 +364,27 @@ impl EmbedOpts { } let mut wasm = if self.dummy { - wit_component::dummy_module(&resolve, world, Mangling::Standard32) + wit_component::dummy_module(&resolve, world, ManglingAndAbi::Standard32) } else if let Some(mangling) = self.dummy_names { - wit_component::dummy_module(&resolve, world, mangling) + wit_component::dummy_module( + &resolve, + world, + match mangling { + Mangling::Standard32 => { + if self.async_callback || self.async_stackful { + bail!("non-legacy mangling not yet supported when generating async dummy modules"); + } + ManglingAndAbi::Standard32 + } + Mangling::Legacy => ManglingAndAbi::Legacy(if self.async_callback { + LiftLowerAbi::AsyncCallback + } else if self.async_stackful { + LiftLowerAbi::AsyncStackful + } else { + LiftLowerAbi::Sync + }), + }, + ) } else { self.io.parse_input_wasm()? };