From 997cd60dd00bf1b4978f5a847e5061ae487a4378 Mon Sep 17 00:00:00 2001 From: Tyler Goodwin Date: Fri, 23 Oct 2026 15:41:18 +1100 Subject: [PATCH 1/7] Add configurable linear memory reservation for the wizening engine --- src/memory.rs | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 src/memory.rs diff --git a/src/memory.rs b/src/memory.rs new file mode 100644 index 0000000..ccb46d2 --- /dev/null +++ b/src/memory.rs @@ -0,0 +1,78 @@ +//! Linear memory configuration for the engine used while wizening. +//! +//! By default Wasmtime reserves 4GiB of virtual address space for every linear +//! memory on 64-bit hosts. That is fine for a build machine in isolation, but +//! it is a problem when the wizening process runs somewhere where address space +//! or snapshotted mappings are accounted for. This module lets callers shrink +//! the reservation with `--memory-reservation` or `WEVAL_MEMORY_RESERVATION`. + +use std::sync::OnceLock; + +use anyhow::{bail, Context}; +use wasmtime::Config; + +/// Environment variable consulted when `--memory-reservation` is not given. +pub(crate) const ENV_VAR: &str = "WEVAL_MEMORY_RESERVATION"; + +static RESERVATION: OnceLock> = OnceLock::new(); + +/// Parse a size such as `64M`, `64MiB`, `1G`, `4096` (bytes) into bytes. +pub(crate) fn parse_size(input: &str) -> anyhow::Result { + let s = input.trim().to_ascii_lowercase(); + let s = s.strip_suffix('b').unwrap_or(&s); + let s = s.strip_suffix('i').unwrap_or(s); + let (digits, shift) = match s.chars().last() { + Some('k') => (&s[..s.len() - 1], 10), + Some('m') => (&s[..s.len() - 1], 20), + Some('g') => (&s[..s.len() - 1], 30), + _ => (s, 0), + }; + let value: u64 = digits + .trim() + .parse() + .with_context(|| format!("invalid memory size: {input:?}"))?; + match value.checked_mul(1u64 << shift) { + Some(bytes) => Ok(bytes), + None => bail!("memory size overflows u64: {input:?}"), + } +} + +/// Record the reservation to use. The flag value wins over the environment. +pub(crate) fn init(flag: Option) -> anyhow::Result<()> { + let value = match flag { + Some(s) => Some(parse_size(&s)?), + None => match std::env::var(ENV_VAR) { + Ok(s) if !s.trim().is_empty() => Some( + parse_size(&s).with_context(|| format!("while reading {ENV_VAR}"))?, + ), + _ => None, + }, + }; + let _ = RESERVATION.set(value); + Ok(()) +} + +/// Apply the configured reservation, if any, to a Wasmtime `Config`. +pub(crate) fn configure(config: &mut Config) { + if let Some(Some(bytes)) = RESERVATION.get() { + config.memory_reservation(*bytes); + config.memory_reservation_for_growth(0); + config.memory_guard_size(0); + config.memory_may_move(true); + } +} + +#[cfg(test)] +mod tests { + use super::parse_size; + + #[test] + fn parses_sizes() { + assert_eq!(parse_size("4096").unwrap(), 4096); + assert_eq!(parse_size("64M").unwrap(), 64 << 20); + assert_eq!(parse_size("64MiB").unwrap(), 64 << 20); + assert_eq!(parse_size("1g").unwrap(), 1 << 30); + assert_eq!(parse_size("512k").unwrap(), 512 << 10); + assert!(parse_size("abc").is_err()); + } +} From 358d26a4cfe13e57d637da20f576d335b3efbc83 Mon Sep 17 00:00:00 2001 From: Tyler Goodwin Date: Fri, 23 Oct 2026 15:41:37 +1100 Subject: [PATCH 2/7] Add --memory-reservation flag and WEVAL_MEMORY_RESERVATION env var --- src/main.rs | 48 ++++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/src/main.rs b/src/main.rs index 3b507e8..f1c66ca 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,6 +17,7 @@ mod filter; mod image; mod intrinsics; mod liveness; +mod memory; mod module; mod state; mod stats; @@ -62,6 +63,13 @@ pub enum Command { #[structopt(long = "allow-wasi-http")] allow_wasi_http: bool, + /// Virtual memory to reserve for each linear memory while wizening, + /// e.g. `64M` or `1G`. Wasmtime reserves 4GiB by default. Can also be + /// set with the WEVAL_MEMORY_RESERVATION environment variable; the flag + /// takes precedence. + #[structopt(long = "memory-reservation")] + memory_reservation: Option, + /// Cache file to use. #[structopt(long = "cache")] cache: Option, @@ -99,29 +107,33 @@ fn main() -> anyhow::Result<()> { allow_wasip2, allow_wasip3, allow_wasi_http, + memory_reservation, cache, cache_ro, show_stats, output_ir, verbose, - } => weval( - input_module, - output_module, - wizen, - preopens, - init_func, - keep_init_func, - cache, - cache_ro, - show_stats, - output_ir, - verbose, - WasiOpts { - p2: allow_wasip2, - p3: allow_wasip3, - http: allow_wasi_http, - }, - ), + } => { + memory::init(memory_reservation)?; + weval( + input_module, + output_module, + wizen, + preopens, + init_func, + keep_init_func, + cache, + cache_ro, + show_stats, + output_ir, + verbose, + WasiOpts { + p2: allow_wasip2, + p3: allow_wasip3, + http: allow_wasi_http, + }, + ) + } } } From f22c38dbf9ec000607f9bfff7c795c751af33c6d Mon Sep 17 00:00:00 2001 From: Tyler Goodwin Date: Fri, 23 Oct 2026 15:41:38 +1100 Subject: [PATCH 3/7] placeholder --- src/component.rs | 260 +---------------------------------------------- 1 file changed, 1 insertion(+), 259 deletions(-) diff --git a/src/component.rs b/src/component.rs index 3f3fc28..311c8dd 100644 --- a/src/component.rs +++ b/src/component.rs @@ -1,259 +1 @@ -use std::path::PathBuf; - -use anyhow::Context; -use wasmparser::{Encoding, Parser, Payload}; -use wasmtime::component::Linker; -use wasmtime::{Config, Engine, Store}; -use wasmtime_wasi::{p2, p3, ResourceTable, WasiCtx, WasiCtxView, WasiView}; -use wasmtime_wasi_http::{p2 as http_p2, p3 as http_p3, WasiHttpCtx}; - -use crate::{cache, configure_wasi}; - -pub(crate) struct Opts { - pub(crate) cache: Option, - pub(crate) cache_ro: Option, - pub(crate) show_stats: bool, - pub(crate) output_ir: Option, - pub(crate) verbose: bool, -} - -#[derive(Clone, Copy, Debug, Default)] -pub(crate) struct WasiOpts { - pub(crate) p2: bool, - pub(crate) p3: bool, - pub(crate) http: bool, -} - -struct ComponentWasiCtx { - wasi: WasiCtx, - http: WasiHttpCtx, - table: ResourceTable, -} - -impl WasiView for ComponentWasiCtx { - fn ctx(&mut self) -> WasiCtxView<'_> { - WasiCtxView { - ctx: &mut self.wasi, - table: &mut self.table, - } - } -} - -impl http_p2::WasiHttpView for ComponentWasiCtx { - fn http(&mut self) -> http_p2::WasiHttpCtxView<'_> { - http_p2::WasiHttpCtxView { - ctx: &mut self.http, - table: &mut self.table, - hooks: Default::default(), - } - } -} - -impl http_p3::WasiHttpView for ComponentWasiCtx { - fn http(&mut self) -> http_p3::WasiHttpCtxView<'_> { - http_p3::WasiHttpCtxView { - ctx: &mut self.http, - table: &mut self.table, - hooks: Default::default(), - } - } -} - -pub(crate) async fn wizen( - component_bytes: Vec, - preopens: Vec, - init_func: String, - keep_init_func: bool, - wasi: WasiOpts, -) -> anyhow::Result> { - let mut config = Config::new(); - config.wasm_bulk_memory(true); - config.wasm_component_model(true); - config.wasm_component_model_async(true); - - let engine = Engine::new(&config)?; - - let mut wasi_ctx = wasmtime_wasi::WasiCtxBuilder::new(); - configure_wasi(&mut wasi_ctx, &preopens)?; - - let ctx = ComponentWasiCtx { - wasi: wasi_ctx.build(), - http: WasiHttpCtx::new(), - table: ResourceTable::new(), - }; - - let mut store = Store::new(&engine, ctx); - let mut linker = Linker::new(&engine); - - if wasi.p2 { - p2::add_to_linker_async(&mut linker)?; - } - if wasi.p3 { - p3::add_to_linker(&mut linker)?; - } - - if wasi.http { - if !wasi.p2 && !wasi.p3 { - anyhow::bail!("--allow-wasi-http requires --allow-wasip2 and/or --allow-wasip3"); - } - if wasi.p2 { - http_p2::add_only_http_to_linker_async(&mut linker)?; - } - if wasi.p3 { - http_p3::add_to_linker(&mut linker)?; - } - } - - let mut wizer = wasmtime_wizer::Wizer::new(); - wizer.init_func(&init_func); - if keep_init_func { - wizer.keep_init_func(true); - } - - Ok(wizer - .run_component(&mut store, &component_bytes, async |store, component| { - linker.instantiate_async(store, component).await - }) - .await?) -} - -pub(crate) fn weval(component_bytes: &[u8], opts: Opts) -> anyhow::Result> { - // This follows the same high-level structure as Wizer's component rewrite: - // See: https://github.com/bytecodealliance/wasmtime/blob/v46.0.1/crates/wizer/src/component/rewrite.rs - let mut encoder = wasm_encoder::Component::new(); - let mut payloads = Parser::new(0).parse_all(component_bytes); - let mut core_module_index = 0u32; - - while let Some(payload) = payloads.next() { - match payload? { - Payload::Version { - encoding: Encoding::Component, - .. - } => {} - Payload::Version { - encoding: Encoding::Module, - .. - } => anyhow::bail!("expected a component, found a core module"), - Payload::ModuleSection { - unchecked_range, .. - } => { - let module_bytes = component_bytes - .get(unchecked_range.clone()) - .context("component core module range is out of bounds")?; - - let rewritten_module = if needs_weval(module_bytes)? { - if opts.verbose { - eprintln!("Processing component core module {core_module_index}..."); - } - - let output_ir = opts - .output_ir - .as_ref() - .map(|dir| dir.join(format!("core-module-{core_module_index}"))); - - crate::module::weval( - module_bytes, - crate::module::Opts { - cache: opts.cache.clone(), - cache_ro: opts.cache_ro.clone(), - module_hash: cache::compute_hash(module_bytes), - show_stats: opts.show_stats, - output_ir, - verbose: opts.verbose, - }, - )? - } else { - module_bytes.to_vec() - }; - - encoder.section(&wasm_encoder::RawSection { - id: wasm_encoder::ComponentSectionId::CoreModule as u8, - data: &rewritten_module, - }); - - core_module_index += 1; - skip_nested(&mut payloads)?; - } - Payload::ComponentSection { - unchecked_range, .. - } => { - let nested_component = component_bytes - .get(unchecked_range.clone()) - .context("nested component range is out of bounds")?; - - encoder.section(&wasm_encoder::RawSection { - id: wasm_encoder::ComponentSectionId::Component as u8, - data: nested_component, - }); - - skip_nested(&mut payloads)?; - } - Payload::End(_) => {} - - payload => { - if let Some((id, range)) = payload.as_section() { - encoder.section(&wasm_encoder::RawSection { - id, - data: component_bytes - .get(range) - .context("component section range is out of bounds")?, - }); - } - } - } - } - - let bytes = encoder.finish(); - - wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::all()) - .validate_all(&bytes) - .map(|_| ()) - .context("rewritten component failed validation")?; - - Ok(bytes) -} - -fn needs_weval(module: &[u8]) -> anyhow::Result { - for payload in Parser::new(0).parse_all(module) { - match payload? { - Payload::ImportSection(imports) => { - for import in imports.into_imports() { - if import?.module == "weval" { - return Ok(true); - } - } - } - Payload::ExportSection(exports) => { - for export in exports { - if export?.name.starts_with("weval.") { - return Ok(true); - } - } - } - Payload::End(_) => break, - _ => {} - } - } - Ok(false) -} - -fn skip_nested<'a>( - payloads: &mut impl Iterator>>, -) -> anyhow::Result<()> { - let mut depth = 1usize; - for payload in payloads { - match payload? { - Payload::ModuleSection { .. } | Payload::ComponentSection { .. } => { - depth += 1; - } - Payload::End(_) => { - depth -= 1; - if depth == 0 { - return Ok(()); - } - } - _ => {} - } - } - anyhow::bail!("nested wasm section did not terminate") -} +PLACEHOLDER \ No newline at end of file From 047d083036cc4079a5970f0276d02dfa39dadebb Mon Sep 17 00:00:00 2001 From: Tyler Goodwin Date: Fri, 23 Oct 2026 15:41:57 +1100 Subject: [PATCH 4/7] Restore component.rs and apply memory reservation config --- src/component.rs | 261 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 260 insertions(+), 1 deletion(-) diff --git a/src/component.rs b/src/component.rs index 311c8dd..832b937 100644 --- a/src/component.rs +++ b/src/component.rs @@ -1 +1,260 @@ -PLACEHOLDER \ No newline at end of file +use std::path::PathBuf; + +use anyhow::Context; +use wasmparser::{Encoding, Parser, Payload}; +use wasmtime::component::Linker; +use wasmtime::{Config, Engine, Store}; +use wasmtime_wasi::{p2, p3, ResourceTable, WasiCtx, WasiCtxView, WasiView}; +use wasmtime_wasi_http::{p2 as http_p2, p3 as http_p3, WasiHttpCtx}; + +use crate::{cache, configure_wasi}; + +pub(crate) struct Opts { + pub(crate) cache: Option, + pub(crate) cache_ro: Option, + pub(crate) show_stats: bool, + pub(crate) output_ir: Option, + pub(crate) verbose: bool, +} + +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct WasiOpts { + pub(crate) p2: bool, + pub(crate) p3: bool, + pub(crate) http: bool, +} + +struct ComponentWasiCtx { + wasi: WasiCtx, + http: WasiHttpCtx, + table: ResourceTable, +} + +impl WasiView for ComponentWasiCtx { + fn ctx(&mut self) -> WasiCtxView<'_> { + WasiCtxView { + ctx: &mut self.wasi, + table: &mut self.table, + } + } +} + +impl http_p2::WasiHttpView for ComponentWasiCtx { + fn http(&mut self) -> http_p2::WasiHttpCtxView<'_> { + http_p2::WasiHttpCtxView { + ctx: &mut self.http, + table: &mut self.table, + hooks: Default::default(), + } + } +} + +impl http_p3::WasiHttpView for ComponentWasiCtx { + fn http(&mut self) -> http_p3::WasiHttpCtxView<'_> { + http_p3::WasiHttpCtxView { + ctx: &mut self.http, + table: &mut self.table, + hooks: Default::default(), + } + } +} + +pub(crate) async fn wizen( + component_bytes: Vec, + preopens: Vec, + init_func: String, + keep_init_func: bool, + wasi: WasiOpts, +) -> anyhow::Result> { + let mut config = Config::new(); + config.wasm_bulk_memory(true); + config.wasm_component_model(true); + config.wasm_component_model_async(true); + crate::memory::configure(&mut config); + + let engine = Engine::new(&config)?; + + let mut wasi_ctx = wasmtime_wasi::WasiCtxBuilder::new(); + configure_wasi(&mut wasi_ctx, &preopens)?; + + let ctx = ComponentWasiCtx { + wasi: wasi_ctx.build(), + http: WasiHttpCtx::new(), + table: ResourceTable::new(), + }; + + let mut store = Store::new(&engine, ctx); + let mut linker = Linker::new(&engine); + + if wasi.p2 { + p2::add_to_linker_async(&mut linker)?; + } + if wasi.p3 { + p3::add_to_linker(&mut linker)?; + } + + if wasi.http { + if !wasi.p2 && !wasi.p3 { + anyhow::bail!("--allow-wasi-http requires --allow-wasip2 and/or --allow-wasip3"); + } + if wasi.p2 { + http_p2::add_only_http_to_linker_async(&mut linker)?; + } + if wasi.p3 { + http_p3::add_to_linker(&mut linker)?; + } + } + + let mut wizer = wasmtime_wizer::Wizer::new(); + wizer.init_func(&init_func); + if keep_init_func { + wizer.keep_init_func(true); + } + + Ok(wizer + .run_component(&mut store, &component_bytes, async |store, component| { + linker.instantiate_async(store, component).await + }) + .await?) +} + +pub(crate) fn weval(component_bytes: &[u8], opts: Opts) -> anyhow::Result> { + // This follows the same high-level structure as Wizer's component rewrite: + // See: https://github.com/bytecodealliance/wasmtime/blob/v46.0.1/crates/wizer/src/component/rewrite.rs + let mut encoder = wasm_encoder::Component::new(); + let mut payloads = Parser::new(0).parse_all(component_bytes); + let mut core_module_index = 0u32; + + while let Some(payload) = payloads.next() { + match payload? { + Payload::Version { + encoding: Encoding::Component, + .. + } => {} + Payload::Version { + encoding: Encoding::Module, + .. + } => anyhow::bail!("expected a component, found a core module"), + Payload::ModuleSection { + unchecked_range, .. + } => { + let module_bytes = component_bytes + .get(unchecked_range.clone()) + .context("component core module range is out of bounds")?; + + let rewritten_module = if needs_weval(module_bytes)? { + if opts.verbose { + eprintln!("Processing component core module {core_module_index}..."); + } + + let output_ir = opts + .output_ir + .as_ref() + .map(|dir| dir.join(format!("core-module-{core_module_index}"))); + + crate::module::weval( + module_bytes, + crate::module::Opts { + cache: opts.cache.clone(), + cache_ro: opts.cache_ro.clone(), + module_hash: cache::compute_hash(module_bytes), + show_stats: opts.show_stats, + output_ir, + verbose: opts.verbose, + }, + )? + } else { + module_bytes.to_vec() + }; + + encoder.section(&wasm_encoder::RawSection { + id: wasm_encoder::ComponentSectionId::CoreModule as u8, + data: &rewritten_module, + }); + + core_module_index += 1; + skip_nested(&mut payloads)?; + } + Payload::ComponentSection { + unchecked_range, .. + } => { + let nested_component = component_bytes + .get(unchecked_range.clone()) + .context("nested component range is out of bounds")?; + + encoder.section(&wasm_encoder::RawSection { + id: wasm_encoder::ComponentSectionId::Component as u8, + data: nested_component, + }); + + skip_nested(&mut payloads)?; + } + Payload::End(_) => {} + + payload => { + if let Some((id, range)) = payload.as_section() { + encoder.section(&wasm_encoder::RawSection { + id, + data: component_bytes + .get(range) + .context("component section range is out of bounds")?, + }); + } + } + } + } + + let bytes = encoder.finish(); + + wasmparser::Validator::new_with_features(wasmparser::WasmFeatures::all()) + .validate_all(&bytes) + .map(|_| ()) + .context("rewritten component failed validation")?; + + Ok(bytes) +} + +fn needs_weval(module: &[u8]) -> anyhow::Result { + for payload in Parser::new(0).parse_all(module) { + match payload? { + Payload::ImportSection(imports) => { + for import in imports.into_imports() { + if import?.module == "weval" { + return Ok(true); + } + } + } + Payload::ExportSection(exports) => { + for export in exports { + if export?.name.starts_with("weval.") { + return Ok(true); + } + } + } + Payload::End(_) => break, + _ => {} + } + } + Ok(false) +} + +fn skip_nested<'a>( + payloads: &mut impl Iterator>>, +) -> anyhow::Result<()> { + let mut depth = 1usize; + for payload in payloads { + match payload? { + Payload::ModuleSection { .. } | Payload::ComponentSection { .. } => { + depth += 1; + } + Payload::End(_) => { + depth -= 1; + if depth == 0 { + return Ok(()); + } + } + _ => {} + } + } + anyhow::bail!("nested wasm section did not terminate") +} From e016380545aedb103068fa1732be5b1a7d351a74 Mon Sep 17 00:00:00 2001 From: Tyler Goodwin Date: Fri, 23 Oct 2026 15:42:06 +1100 Subject: [PATCH 5/7] Apply memory reservation config to the core module wizening engine --- src/module.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/module.rs b/src/module.rs index b7470f0..7649278 100644 --- a/src/module.rs +++ b/src/module.rs @@ -24,6 +24,7 @@ pub(crate) async fn wizen( ) -> anyhow::Result> { let mut config = Config::new(); config.wasm_bulk_memory(true); + crate::memory::configure(&mut config); let engine = Engine::new(&config)?; let mut wasi_ctx = wasmtime_wasi::WasiCtxBuilder::new(); From a03ff2e04a88814adbafc5dd9b58878de7d07514 Mon Sep 17 00:00:00 2001 From: Tyler Goodwin Date: Mon, 26 Oct 2026 09:27:20 +1100 Subject: [PATCH 6/7] Reword memory module docs to be environment agnostic --- src/memory.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/memory.rs b/src/memory.rs index ccb46d2..c5658ff 100644 --- a/src/memory.rs +++ b/src/memory.rs @@ -1,10 +1,10 @@ //! Linear memory configuration for the engine used while wizening. //! //! By default Wasmtime reserves 4GiB of virtual address space for every linear -//! memory on 64-bit hosts. That is fine for a build machine in isolation, but -//! it is a problem when the wizening process runs somewhere where address space -//! or snapshotted mappings are accounted for. This module lets callers shrink -//! the reservation with `--memory-reservation` or `WEVAL_MEMORY_RESERVATION`. +//! memory on 64-bit hosts. This is primarily useful for configuring the virtual +//! address space when running in a resource constrained environment. The +//! reservation can be overridden with `--memory-reservation` or the +//! `WEVAL_MEMORY_RESERVATION` environment variable. use std::sync::OnceLock; From ccc508945f8394171236461e778802a57afacb63 Mon Sep 17 00:00:00 2001 From: Tyler Goodwin Date: Mon, 26 Oct 2026 09:50:32 +1100 Subject: [PATCH 7/7] cargo fmt --- src/memory.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/memory.rs b/src/memory.rs index c5658ff..91434ed 100644 --- a/src/memory.rs +++ b/src/memory.rs @@ -42,9 +42,9 @@ pub(crate) fn init(flag: Option) -> anyhow::Result<()> { let value = match flag { Some(s) => Some(parse_size(&s)?), None => match std::env::var(ENV_VAR) { - Ok(s) if !s.trim().is_empty() => Some( - parse_size(&s).with_context(|| format!("while reading {ENV_VAR}"))?, - ), + Ok(s) if !s.trim().is_empty() => { + Some(parse_size(&s).with_context(|| format!("while reading {ENV_VAR}"))?) + } _ => None, }, };