Skip to content

Commit

Permalink
feat(wasm-builder): Successful build returns paths to the wasm and op…
Browse files Browse the repository at this point in the history 
…t wasm files (#4209)
  • Loading branch information
@vobradovich
vobradovich authored Sep 2, 2024
1 parent 7ad7cfb commit e36bfb7
Show file tree
Hide file tree
Showing 3 changed files with 97 additions and 82 deletions.
63 changes: 41 additions & 22 deletions utils/wasm-builder/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,22 +100,29 @@ impl WasmBuilder {
}

/// Build the program and produce an output WASM binary.
pub fn build(self) {
///
/// Returns `None` if `__GEAR_WASM_BUILDER_NO_BUILD` flag is set.
/// Returns `Some(_)` with a tuple of paths to wasm & opt wasm file
/// if the build was successful.
pub fn build(self) -> Option<(PathBuf, PathBuf)> {
if env::var("__GEAR_WASM_BUILDER_NO_BUILD").is_ok() || is_intellij_sync() {
self.wasm_project.provide_dummy_wasm_binary_if_not_exist();
return;
_ = self.wasm_project.provide_dummy_wasm_binary_if_not_exist();
return None;
}

if let Err(e) = self.build_project() {
eprintln!("error: {e}");
e.chain()
.skip(1)
.for_each(|cause| eprintln!("| {cause}"));
process::exit(1);
match self.build_project() {
Err(e) => {
eprintln!("error: {e}");
e.chain()
.skip(1)
.for_each(|cause| eprintln!("| {cause}"));
process::exit(1);
}
Ok(r) => r,
}
}

fn build_project(mut self) -> Result<()> {
fn build_project(mut self) -> Result<Option<(PathBuf, PathBuf)>> {
self.wasm_project.generate()?;

self.cargo
Expand Down Expand Up @@ -236,46 +243,58 @@ fn is_intellij_sync() -> bool {
const FEATURES_TO_EXCLUDE_BY_DEFAULT: &[&str] = &["std"];

/// Shorthand function to be used in `build.rs`.
pub fn build() {
///
/// See [WasmBuilder::build()].
pub fn build() -> Option<(PathBuf, PathBuf)> {
WasmBuilder::new()
.exclude_features(FEATURES_TO_EXCLUDE_BY_DEFAULT.to_vec())
.build();
.build()
}

/// Shorthand function to be used in `build.rs`.
pub fn build_with_metadata<T: Metadata>() {
///
/// See [WasmBuilder::build()].
pub fn build_with_metadata<T: Metadata>() -> Option<(PathBuf, PathBuf)> {
WasmBuilder::with_meta(T::repr())
.exclude_features(FEATURES_TO_EXCLUDE_BY_DEFAULT.to_vec())
.build();
.build()
}

/// Shorthand function to be used in `build.rs`.
pub fn build_metawasm() {
///
/// See [WasmBuilder::build()].
pub fn build_metawasm() -> Option<(PathBuf, PathBuf)> {
WasmBuilder::new_metawasm()
.exclude_features(FEATURES_TO_EXCLUDE_BY_DEFAULT.to_vec())
.build();
.build()
}

/// Shorthand function to be used in `build.rs`.
pub fn recommended_nightly() {
///
/// See [WasmBuilder::build()].
pub fn recommended_nightly() -> Option<(PathBuf, PathBuf)> {
WasmBuilder::new()
.exclude_features(FEATURES_TO_EXCLUDE_BY_DEFAULT.to_vec())
.with_recommended_toolchain()
.build();
.build()
}

/// Shorthand function to be used in `build.rs`.
pub fn recommended_nightly_with_metadata<T: Metadata>() {
///
/// See [WasmBuilder::build()].
pub fn recommended_nightly_with_metadata<T: Metadata>() -> Option<(PathBuf, PathBuf)> {
WasmBuilder::with_meta(T::repr())
.exclude_features(FEATURES_TO_EXCLUDE_BY_DEFAULT.to_vec())
.with_recommended_toolchain()
.build();
.build()
}

/// Shorthand function to be used in `build.rs`.
pub fn recommended_nightly_metawasm() {
///
/// See [WasmBuilder::build()].
pub fn recommended_nightly_metawasm() -> Option<(PathBuf, PathBuf)> {
WasmBuilder::new_metawasm()
.exclude_features(FEATURES_TO_EXCLUDE_BY_DEFAULT.to_vec())
.with_recommended_toolchain()
.build();
.build()
}
84 changes: 40 additions & 44 deletions utils/wasm-builder/src/wasm_project.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.

use crate::{code_validator::CodeValidator, crate_info::CrateInfo, smart_fs};
use anyhow::{Context, Result};
use anyhow::{Context, Ok, Result};
use chrono::offset::Local as ChronoLocal;
use gear_wasm_optimizer::{self as optimize, OptType, Optimizer};
use gmeta::MetadataRepr;
Expand Down Expand Up @@ -335,7 +335,7 @@ extern "C" fn metahash() {{
pub const WASM_BINARY: &[u8] = include_bytes!("{}");
#[allow(unused)]
pub const WASM_EXPORTS: &[&str] = &{:?};"#,
display_path(meta_wasm_path.clone()),
display_path(meta_wasm_path.as_path()),
Self::get_exports(&meta_wasm_path)?,
),
)
Expand Down Expand Up @@ -367,45 +367,40 @@ extern "C" fn metahash() {{
/// Generates output optimized wasm file, `.binpath` file for our tests
/// system and wasm binaries informational file.
/// Makes a copy of original wasm file in `self.wasm_target_dir`.
pub fn postprocess_opt(
pub fn postprocess_opt<P: AsRef<Path>>(
&self,
original_wasm_path: &PathBuf,
original_wasm_path: P,
file_base_name: &String,
) -> Result<()> {
) -> Result<PathBuf> {
let [original_copy_wasm_path, opt_wasm_path] = [".wasm", ".opt.wasm"]
.map(|ext| self.wasm_target_dir.join([file_base_name, ext].concat()));

// Copy original file to `self.wasm_target_dir`
smart_fs::copy_if_newer(original_wasm_path, &original_copy_wasm_path)
smart_fs::copy_if_newer(&original_wasm_path, &original_copy_wasm_path)
.context("unable to copy WASM file")?;

// Optimize wasm using and `wasm-opt` and our optimizations.
if smart_fs::check_if_newer(original_wasm_path, &opt_wasm_path)? {
let path = optimize::optimize_wasm(
original_copy_wasm_path.clone(),
opt_wasm_path.clone(),
"4",
true,
)
.map(|res| {
log::info!(
"Wasm-opt reduced wasm size: {} -> {}",
res.original_size,
res.optimized_size
);
opt_wasm_path.clone()
})
.unwrap_or_else(|err| {
println!("cargo:warning=wasm-opt optimizations error: {}", err);
original_copy_wasm_path.clone()
});
if smart_fs::check_if_newer(&original_wasm_path, &opt_wasm_path)? {
let path = optimize::optimize_wasm(&original_copy_wasm_path, &opt_wasm_path, "4", true)
.map(|res| {
log::info!(
"Wasm-opt reduced wasm size: {} -> {}",
res.original_size,
res.optimized_size
);
opt_wasm_path.clone()
})
.unwrap_or_else(|err| {
println!("cargo:warning=wasm-opt optimizations error: {}", err);
original_copy_wasm_path.clone()
});

let mut optimizer = Optimizer::new(path)?;
optimizer
.insert_stack_end_export()
.unwrap_or_else(|err| log::info!("Cannot insert stack end export: {}", err));
optimizer.strip_custom_sections();
fs::write(opt_wasm_path.clone(), optimizer.optimize(OptType::Opt)?)
fs::write(&opt_wasm_path, optimizer.optimize(OptType::Opt)?)
.context("Failed to write optimized WASM binary")?;
}

Expand All @@ -428,12 +423,13 @@ extern "C" fn metahash() {{
#[allow(unused)]
pub const WASM_BINARY_OPT: &[u8] = include_bytes!("{}");
{}"#,
display_path(original_copy_wasm_path),
display_path(opt_wasm_path),
display_path(original_copy_wasm_path.as_path()),
display_path(opt_wasm_path.as_path()),
metadata,
),
)
.context("unable to write `wasm_binary.rs`")
.context("unable to write `wasm_binary.rs`")?;
Ok(opt_wasm_path)
}

/// Post-processing after the WASM binary has been built.
Expand All @@ -442,7 +438,7 @@ extern "C" fn metahash() {{
/// `target/wasm32-unknown-unknown/<profile>`
/// - Generate optimized and metadata WASM binaries from the built program
/// - Generate `wasm_binary.rs` source file in `OUT_DIR`
pub fn postprocess(&self) -> Result<()> {
pub fn postprocess(&self) -> Result<Option<(PathBuf, PathBuf)>> {
let file_base_name = self
.file_base_name
.as_ref()
Expand Down Expand Up @@ -508,15 +504,18 @@ extern "C" fn metahash() {{
}
}

// Tuple with PathBuf last wasm & opt.wasm
let mut wasm_paths: Option<(PathBuf, PathBuf)> = None;
for (wasm_path, file_base_name) in &wasm_files {
if self.project_type.is_metawasm() {
self.postprocess_meta(wasm_path, file_base_name)?;
} else {
self.postprocess_opt(wasm_path, file_base_name)?;
let wasm_opt = self.postprocess_opt(wasm_path, file_base_name)?;
wasm_paths = Some((wasm_path.clone(), wasm_opt));
}
}

for (wasm_path, _) in wasm_files {
for (wasm_path, _) in &wasm_files {
let code = fs::read(wasm_path)?;
let validator = CodeValidator::try_from(code)?;

Expand All @@ -530,8 +529,7 @@ extern "C" fn metahash() {{
if env::var("__GEAR_WASM_BUILDER_NO_FEATURES_TRACKING").is_err() {
self.force_rerun_on_next_run(&original_wasm_path)?;
}

Ok(())
Ok(wasm_paths)
}

fn get_exports(file: &PathBuf) -> Result<Vec<String>> {
Expand Down Expand Up @@ -566,10 +564,10 @@ extern "C" fn metahash() {{
}

/// Provide a dummy WASM binary if there doesn't exist one.
pub fn provide_dummy_wasm_binary_if_not_exist(&self) {
pub fn provide_dummy_wasm_binary_if_not_exist(&self) -> PathBuf {
let wasm_binary_rs = self.out_dir.join("wasm_binary.rs");
if wasm_binary_rs.exists() {
return;
return wasm_binary_rs;
}

let content = if !self.project_type.is_metawasm() {
Expand All @@ -586,17 +584,15 @@ extern "C" fn metahash() {{
pub const WASM_EXPORTS: &[&str] = &[];
"#
};
fs::write(wasm_binary_rs.as_path(), content).unwrap_or_else(|_| {
panic!(
"Writing `{}` should not fail!",
display_path(wasm_binary_rs)
)
});
let path = wasm_binary_rs.as_path();
fs::write(path, content)
.unwrap_or_else(|_| panic!("Writing `{}` should not fail!", display_path(path)));
wasm_binary_rs
}
}

// Windows has path like `path\to\somewhere` which is incorrect for `include_*`
// Rust's macros
fn display_path(path: PathBuf) -> String {
path.display().to_string().replace('\\', "/")
fn display_path<P: AsRef<Path>>(path: P) -> String {
path.as_ref().display().to_string().replace('\\', "/")
}
32 changes: 16 additions & 16 deletions utils/wasm-optimizer/src/optimize.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,8 @@ use pwasm_utils::{
#[cfg(not(feature = "wasm-opt"))]
use std::process::Command;
use std::{
ffi::OsStr,
fs::{self, metadata},
path::PathBuf,
path::{Path, PathBuf},
};

#[cfg(feature = "wasm-opt")]
Expand Down Expand Up @@ -149,29 +148,30 @@ pub struct OptimizationResult {
///
/// The intention is to reduce the size of bloated Wasm binaries as a result of
/// missing optimizations (or bugs?) between Rust and Wasm.
pub fn optimize_wasm(
source: PathBuf,
destination: PathBuf,
pub fn optimize_wasm<P: AsRef<Path>>(
source: P,
destination: P,
optimization_passes: &str,
keep_debug_symbols: bool,
) -> Result<OptimizationResult> {
let original_size = metadata(&source)?.len() as f64 / 1000.0;

do_optimization(
source.as_os_str(),
destination.as_os_str(),
&source,
&destination,
optimization_passes,
keep_debug_symbols,
)?;

let destination = destination.as_ref();
if !destination.exists() {
return Err(anyhow::anyhow!(
"Optimization failed, optimized wasm output file `{}` not found.",
destination.display()
));
}

let optimized_size = metadata(&destination)?.len() as f64 / 1000.0;
let optimized_size = metadata(destination)?.len() as f64 / 1000.0;

Ok(OptimizationResult {
original_size,
Expand All @@ -187,9 +187,9 @@ pub fn optimize_wasm(
/// resulting in potentially a lot of time spent optimizing.
///
/// If successful, the optimized Wasm is written to `dest_optimized`.
pub fn do_optimization(
dest_wasm: &OsStr,
dest_optimized: &OsStr,
pub fn do_optimization<P: AsRef<Path>>(
dest_wasm: P,
dest_optimized: P,
optimization_level: &str,
keep_debug_symbols: bool,
) -> Result<()> {
Expand Down Expand Up @@ -221,10 +221,10 @@ pub fn do_optimization(
);
let mut command = Command::new(wasm_opt_path);
command
.arg(dest_wasm)
.arg(dest_wasm.as_ref())
.arg(format!("-O{optimization_level}"))
.arg("-o")
.arg(dest_optimized)
.arg(dest_optimized.as_ref())
.arg("-mvp")
.arg("--enable-sign-ext")
// the memory in our module is imported, `wasm-opt` needs to be told that
Expand Down Expand Up @@ -259,9 +259,9 @@ pub fn do_optimization(
/// resulting in potentially a lot of time spent optimizing.
///
/// If successful, the optimized Wasm is written to `dest_optimized`.
pub fn do_optimization(
dest_wasm: &OsStr,
dest_optimized: &OsStr,
pub fn do_optimization<P: AsRef<Path>>(
dest_wasm: P,
dest_optimized: P,
optimization_level: &str,
keep_debug_symbols: bool,
) -> Result<()> {
Expand Down

0 comments on commit e36bfb7

Please sign in to comment.