From 0bdcb5c07070454c006195f31567251c7f6ec5e4 Mon Sep 17 00:00:00 2001 From: Christoph Burgdorf Date: Wed, 8 Jul 2026 09:53:22 +0300 Subject: [PATCH] Add fe build --from-metadata to rebuild from a metadata.json input `fe build --emit metadata` produces a complete, deterministic recompilation input, but the reverse step existed only as a test helper. External toolchains (foundry-compilers, Sourcify-style verifiers) need it as a CLI feature: one invocation, JSON in, artifacts out. `fe build --from-metadata ` (`-` reads stdin) materializes the recorded project (regenerated fe.toml per settings.ingots[] entry, sources routed by namespace) into a temporary directory and runs the normal ingot build on it: - default target is the settings.compilationTarget contract; --contract overrides - optimizer level comes from settings.optimizer.level; an explicit -O wins with a stderr warning that the bytecode deviates - compiler.version mismatch warns on stderr without aborting - artifacts follow --emit into --out-dir, defaulting to ./out - invalid input (bad JSON, language != "Fe", missing sources/ingots) exits 1; combining with [path], --ingot, --standalone, or --report is a clap conflict (exit 2) The reconstruction lives in crates/fe/src/metadata_input.rs and validates untrusted input (no path traversal out of the temp dir). The round-trip tests now exercise the CLI feature directly; the test helper is removed. New tests cover stdin, target selection, the -O and version warnings, and the error cases. --- CLI.md | 30 ++ crates/fe/src/build.rs | 131 ++++++++ crates/fe/src/main.rs | 25 ++ crates/fe/src/metadata_input.rs | 392 ++++++++++++++++++++++++ crates/fe/tests/cli_output.rs | 514 ++++++++++++++++++++++++++------ newsfragments/1505.feature.md | 1 + 6 files changed, 997 insertions(+), 96 deletions(-) create mode 100644 crates/fe/src/metadata_input.rs create mode 100644 newsfragments/1505.feature.md diff --git a/CLI.md b/CLI.md index 987a540788..8f55b20696 100644 --- a/CLI.md +++ b/CLI.md @@ -94,6 +94,7 @@ Builds use the Sonatina codegen pipeline and generate EVM bytecode directly. ``` fe build [--standalone] [--contract ] [--optimize ] [--out-dir ] [--report [--report-out ] [--report-failed-only]] [path] +fe build --from-metadata [--contract ] [--optimize ] [--out-dir ] [--emit ] ``` If `path` is omitted, it defaults to `.`. @@ -234,6 +235,35 @@ Filenames are “sanitized” from contract names: This sanitization is also what the workspace collision check uses. +### Rebuilding from metadata: `--from-metadata` + +`fe build --from-metadata ` rebuilds a contract from a `.metadata.json` +recompilation input (as produced by `--emit metadata`) instead of a source tree. `-` reads the +JSON from stdin, so external toolchains can pipe a synthesized metadata document into a single +`fe build` invocation. All diagnostics go to stderr. + +Behavior: + +- The recorded project (all `sources`, one regenerated `fe.toml` per `settings.ingots[]` entry) is + materialized into a temporary directory and built through the normal ingot build path; the + bundled `std`/`core` are provided by the compiler. +- **Contract selection**: defaults to the contract recorded in `settings.compilationTarget`; + `--contract ` overrides it. +- **Optimizer level**: defaults to `settings.optimizer.level`; an explicit `-O`/`--optimize` wins, + with a warning on stderr that the rebuilt bytecode will not match the verified artifact. +- **Arithmetic**: the per-ingot effective `arithmetic` values from the metadata are applied via + the regenerated `fe.toml` files (`dependency-arithmetic` is deliberately not re-applied; the + metadata records post-forcing values). +- **Version check**: if `compiler.version` differs from the running compiler, a warning is printed + to stderr (no abort); exact bytecode reproduction is only guaranteed with the same version. +- **Output**: artifacts are selected with `--emit` as usual and written to `--out-dir`, which + defaults to `./out` (relative to the current working directory, since there is no project + directory to anchor to). + +Errors (exit code 1): unreadable input, invalid JSON, `language != "Fe"`, missing `sources`, or a +missing/rootless `settings.ingots`. Combining `--from-metadata` with the `[path]` argument, +`--ingot`, `--standalone`, or `--report` is a CLI usage error (exit code 2). + ### Reports: `--report` `fe build` can optionally write a `.tar.gz` debugging report (useful for sharing failures): diff --git a/crates/fe/src/build.rs b/crates/fe/src/build.rs index 026a56d19f..b7837f4cdf 100644 --- a/crates/fe/src/build.rs +++ b/crates/fe/src/build.rs @@ -306,6 +306,137 @@ pub fn build( } } +/// `fe build --from-metadata`: rebuild from a `metadata.json` recompilation +/// input instead of a source tree. Reads the metadata (from a file, or stdin +/// for `-`), materializes the recorded project into a temporary directory, +/// and runs the normal ingot build on it. Exits the process on failure. +pub fn build_from_metadata( + input: &Utf8Path, + contract: Option<&str>, + optimize: Option<&str>, + emit: &[BuildEmit], + out_dir: Option<&Utf8PathBuf>, + profile: &str, + use_recovery_mode: bool, +) { + let emit = EmitSelection::from_requested(emit); + + let metadata = match crate::metadata_input::read_metadata(input) { + Ok(metadata) => metadata, + Err(err) => { + eprintln!("Error: {err}"); + std::process::exit(1); + } + }; + if let Err(err) = crate::metadata_input::validate_metadata(&metadata) { + eprintln!("Error: {err}"); + std::process::exit(1); + } + + // Exact reproduction is only guaranteed with the same compiler version. + if let Some(version) = metadata["compiler"]["version"].as_str() { + let running = env!("CARGO_PKG_VERSION"); + if version != running { + eprintln!( + "Warning: metadata was produced by fe {version}, but this is fe {running}; \ + the rebuilt bytecode may differ from the original" + ); + } + } + // Non-release builds share a package version; the recorded commit pins the + // exact source revision. + if let (Some(recorded), Some(running)) = ( + metadata["compiler"]["commit"].as_str(), + option_env!("FE_GIT_HASH").filter(|hash| !hash.is_empty()), + ) && recorded != running + { + eprintln!( + "Warning: metadata was produced by fe commit {recorded}, but this is fe commit \ + {running}; the rebuilt bytecode may differ from the original" + ); + } + + // The metadata's optimizer level is the default; an explicit `-O` wins. + let recorded_level = metadata["settings"]["optimizer"]["level"].as_str(); + let level = match (optimize, recorded_level) { + (Some(flag), Some(recorded)) if flag != recorded => { + eprintln!( + "Warning: -O {flag} overrides optimizer level {recorded} recorded in the \ + metadata; the rebuilt bytecode will not match the verified artifact" + ); + flag + } + (Some(flag), _) => flag, + (None, Some(recorded)) => recorded, + (None, None) => "1", + }; + let opt_level: OptLevel = match level.parse() { + Ok(level) => level, + Err(err) => { + eprintln!("Error: {err}"); + std::process::exit(1); + } + }; + + // Default target: the contract recorded in `compilationTarget`; an + // explicit `--contract` overrides it. + let contract = contract.map(str::to_string).or_else(|| { + metadata["settings"]["compilationTarget"] + .as_object() + .and_then(|target| target.values().next()) + .and_then(|name| name.as_str()) + .map(str::to_string) + }); + + let temp = match tempfile::Builder::new() + .prefix("fe-from-metadata") + .tempdir() + { + Ok(temp) => temp, + Err(err) => { + eprintln!("Error: Failed to create temporary project directory: {err}"); + std::process::exit(1); + } + }; + let Some(temp_root) = Utf8Path::from_path(temp.path()) else { + eprintln!("Error: temporary project directory path is not valid UTF-8"); + std::process::exit(1); + }; + let root_dir = match crate::metadata_input::reconstruct_project(&metadata, temp_root) { + Ok(dir) => dir, + Err(err) => { + eprintln!("Error: {err}"); + std::process::exit(1); + } + }; + + let out_dir = out_dir.cloned().unwrap_or_else(|| Utf8PathBuf::from("out")); + + let mut db = DriverDataBase::default(); + db.compiler_options() + .set_recovery_mode(&mut db) + .to(use_recovery_mode); + db.compilation_settings() + .set_profile(&mut db) + .to(profile.into()); + + let had_errors = build_directory( + &mut db, + &root_dir, + None, + contract.as_deref(), + opt_level, + emit, + Some(&out_dir), + None, + ); + + drop(temp); + if had_errors { + std::process::exit(1); + } +} + #[allow(clippy::too_many_arguments)] fn build_file( db: &mut DriverDataBase, diff --git a/crates/fe/src/main.rs b/crates/fe/src/main.rs index b26c83843a..334ed91c07 100644 --- a/crates/fe/src/main.rs +++ b/crates/fe/src/main.rs @@ -7,6 +7,7 @@ mod dependency_diagnostics; mod doc; #[cfg(feature = "doc-server")] mod doc_serve; +mod metadata_input; mod report; mod test; #[cfg(not(target_arch = "wasm32"))] @@ -111,6 +112,17 @@ pub enum Command { /// Treat a `.fe` file target as standalone, even if it is inside an ingot. #[arg(long)] standalone: bool, + /// Rebuild from a `.metadata.json` recompilation input produced by + /// `--emit metadata` (`-` reads the JSON from stdin). + /// + /// The recorded project is materialized into a temporary directory and built with + /// the settings captured in the metadata. Artifacts default to `./out`. + #[arg( + long, + value_name = "PATH", + conflicts_with_all = ["path", "ingot", "standalone", "report"] + )] + from_metadata: Option, /// Build a specific contract by name (defaults to all contracts in the target). #[arg(long)] contract: Option, @@ -456,6 +468,7 @@ pub fn run(opts: &Options) { path, ingot, standalone, + from_metadata, contract, optimize, out_dir, @@ -466,6 +479,18 @@ pub fn run(opts: &Options) { report_failed_only, recovery_mode, } => { + if let Some(metadata_path) = from_metadata { + build::build_from_metadata( + metadata_path, + contract.as_deref(), + optimize.as_deref(), + emit, + out_dir.as_ref(), + profile, + *recovery_mode, + ); + return; + } let opt_level = match effective_opt_level(optimize.as_deref()) { Ok(level) => level, Err(err) => { diff --git a/crates/fe/src/metadata_input.rs b/crates/fe/src/metadata_input.rs new file mode 100644 index 0000000000..a392774c10 --- /dev/null +++ b/crates/fe/src/metadata_input.rs @@ -0,0 +1,392 @@ +//! Input side of the `metadata.json` round trip (`fe build --from-metadata`). +//! +//! Parses and validates a contract metadata artifact produced by +//! `fe build --emit metadata` (see `write_metadata_artifacts` in `build.rs`) +//! and materializes the recorded project so the normal build path can +//! recompile it. + +use std::{fs, io::Read}; + +use camino::{Utf8Path, Utf8PathBuf}; +use serde_json::Value; + +/// Read metadata JSON from `input`; `-` reads from stdin. +pub fn read_metadata(input: &Utf8Path) -> Result { + let (raw, source) = if input.as_str() == "-" { + let mut buf = String::new(); + std::io::stdin() + .read_to_string(&mut buf) + .map_err(|err| format!("Failed to read metadata from stdin: {err}"))?; + (buf, "".to_string()) + } else { + ( + fs::read_to_string(input.as_std_path()) + .map_err(|err| format!("Failed to read metadata file {input}: {err}"))?, + input.to_string(), + ) + }; + serde_json::from_str(&raw) + .map_err(|err| format!("Failed to parse metadata JSON from {source}: {err}")) +} + +/// Check the fields the reconstruction relies on, with actionable messages. +pub fn validate_metadata(metadata: &Value) -> Result<(), String> { + match metadata["language"].as_str() { + Some("Fe") => {} + Some(other) => { + return Err(format!( + "metadata `language` must be \"Fe\", found \"{other}\"" + )); + } + None => return Err("metadata is missing the `language` field".to_string()), + } + let sources = metadata["sources"] + .as_object() + .ok_or_else(|| "metadata is missing the `sources` object".to_string())?; + if sources.is_empty() { + return Err("metadata `sources` is empty".to_string()); + } + let ingots = metadata["settings"]["ingots"] + .as_array() + .ok_or_else(|| "metadata is missing the `settings.ingots` array".to_string())?; + let root_count = ingots + .iter() + .filter(|ingot| ingot["namespace"].as_str().unwrap_or("").is_empty()) + .count(); + if root_count != 1 { + return Err(format!( + "metadata `settings.ingots` must contain exactly one root entry (empty namespace), found {root_count}" + )); + } + Ok(()) +} + +/// Materialize a buildable project under `dest` from validated metadata: one +/// directory per `settings.ingots[]` entry (root at `dest/`, deps at +/// `dest/`), regenerating each `fe.toml` and writing every +/// `sources` entry into its namespaced layout. Returns the root ingot +/// directory. +pub fn reconstruct_project(metadata: &Value, dest: &Utf8Path) -> Result { + let ingots = metadata["settings"]["ingots"] + .as_array() + .ok_or_else(|| "metadata is missing the `settings.ingots` array".to_string())?; + + // Non-root namespaces, used to route each source key to its owning ingot. + let mut non_root_namespaces: Vec = Vec::new(); + for ingot in ingots { + let namespace = ingot["namespace"].as_str().unwrap_or(""); + if !namespace.is_empty() { + checked_dir_name(namespace, "ingot namespace")?; + non_root_namespaces.push(namespace.to_string()); + } + } + + // The root ingot directory is named after the root ingot; dodge the + // (unlikely) case of a dependency namespace with the same name. Nothing + // references the root directory by name, so renaming it is safe. + let root = ingots + .iter() + .find(|ingot| ingot["namespace"].as_str().unwrap_or("").is_empty()) + .ok_or_else(|| { + "metadata `settings.ingots` has no root entry (empty namespace)".to_string() + })?; + let root_name = root["name"] + .as_str() + .ok_or_else(|| "root `settings.ingots` entry is missing `name`".to_string())?; + // Emitted names can be arbitrary directory basenames (e.g. `my%20project` + // for a standalone target in a URL-encoded path), so sanitize rather than + // validate; unlike namespaces, the name never has to survive as-is. + let mut root_dir_name = sanitize_ingot_name(root_name); + while non_root_namespaces.contains(&root_dir_name) { + root_dir_name.push_str("-root"); + } + + let dir_for = |namespace: &str| -> Utf8PathBuf { + if namespace.is_empty() { + dest.join(&root_dir_name) + } else { + dest.join(namespace) + } + }; + + // 1. Regenerate each ingot's fe.toml. + let empty_deps = serde_json::Map::new(); + for ingot in ingots { + let name = ingot["name"] + .as_str() + .ok_or_else(|| "`settings.ingots` entry is missing `name`".to_string())?; + let namespace = ingot["namespace"].as_str().unwrap_or(""); + let dir = dir_for(namespace); + fs::create_dir_all(dir.join("src").as_std_path()) + .map_err(|err| format!("Failed to create {dir}/src: {err}"))?; + + let mut toml = format!("[ingot]\nname = \"{}\"\n", sanitize_ingot_name(name)); + if let Some(version) = ingot["version"].as_str() { + checked_version(version)?; + toml.push_str(&format!("version = \"{version}\"\n")); + } else { + // Standalone-target metadata records `version: null` (the pseudo-ingot + // has none), but fe.toml requires a version; it does not affect codegen. + toml.push_str("version = \"0.1.0\"\n"); + } + if let Some(arith) = ingot["arithmetic"].as_str() { + if arith != "checked" && arith != "unchecked" { + return Err(format!( + "invalid `arithmetic` value \"{arith}\" for ingot `{name}` in metadata" + )); + } + toml.push_str(&format!("arithmetic = \"{arith}\"\n")); + } + let deps = ingot["dependencies"].as_object().unwrap_or(&empty_deps); + let path_deps: Vec<(&String, &str)> = deps + .iter() + .filter_map(|(alias, target)| { + let target = target.as_str()?; + // std/core are provided by the compiler; only scaffold real path deps. + (target != "std" && target != "core").then_some((alias, target)) + }) + .collect(); + if !path_deps.is_empty() { + toml.push_str("\n[dependencies]\n"); + for (alias, target) in path_deps { + checked_dependency_alias(alias)?; + if !non_root_namespaces.iter().any(|ns| ns == target) { + return Err(format!( + "dependency `{alias}` of ingot `{name}` points at unknown namespace \"{target}\"" + )); + } + toml.push_str(&format!("{alias} = {{ path = \"../{target}\" }}\n")); + } + } + fs::write(dir.join("fe.toml").as_std_path(), toml) + .map_err(|err| format!("Failed to write {dir}/fe.toml: {err}"))?; + } + + // 2. Write every source into its owning ingot's namespaced src/ layout. + let sources = metadata["sources"] + .as_object() + .ok_or_else(|| "metadata is missing the `sources` object".to_string())?; + for (key, source) in sources { + let content = source["content"] + .as_str() + .ok_or_else(|| format!("metadata source \"{key}\" is missing string `content`"))?; + let (namespace, rel) = non_root_namespaces + .iter() + .find_map(|ns| { + key.strip_prefix(&format!("{ns}/")) + .map(|rel| (ns.as_str(), rel)) + }) + .unwrap_or(("", key.as_str())); + checked_source_path(rel)?; + // Standalone-target metadata keys its single source by file basename + // (no directory); the ingot resolver only loads `src/**/*.fe`, so + // materialize bare keys under `src/`. + let target = if rel.contains('/') { + dir_for(namespace).join(rel) + } else { + dir_for(namespace).join("src").join(rel) + }; + if let Some(parent) = target.parent() { + fs::create_dir_all(parent.as_std_path()) + .map_err(|err| format!("Failed to create {parent}: {err}"))?; + } + fs::write(target.as_std_path(), content) + .map_err(|err| format!("Failed to write {target}: {err}"))?; + } + + Ok(dest.join(root_dir_name)) +} + +/// Ingot names and namespaces become directory names and TOML string values, +/// so restrict them to the alphabet the metadata emitter produces. +fn checked_dir_name(name: &str, what: &str) -> Result<(), String> { + let valid = !name.is_empty() + && name != "." + && name != ".." + && name + .chars() + .all(|c| c.is_alphanumeric() || matches!(c, '_' | '-' | '.')); + if valid { + Ok(()) + } else { + Err(format!("invalid {what} \"{name}\" in metadata")) + } +} + +/// The emitter falls back to directory basenames for unnamed ingots (e.g. a +/// standalone target in a temp dir), which may not be valid Fe ingot names +/// (alphanumeric + `_`). The name does not affect codegen, so sanitize it for +/// the regenerated `fe.toml`. +fn sanitize_ingot_name(name: &str) -> String { + if name.is_empty() { + return "ingot".to_string(); + } + name.chars() + .map(|c| { + if c.is_alphanumeric() || c == '_' { + c + } else { + '_' + } + }) + .collect() +} + +/// Versions are written into `fe.toml` string values; allow the SemVer alphabet. +fn checked_version(version: &str) -> Result<(), String> { + let valid = version + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '+' | '_')); + if valid { + Ok(()) + } else { + Err(format!("invalid ingot version \"{version}\" in metadata")) + } +} + +/// Dependency aliases become bare `fe.toml` keys. +fn checked_dependency_alias(alias: &str) -> Result<(), String> { + let valid = !alias.is_empty() + && alias + .chars() + .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-')); + if valid { + Ok(()) + } else { + Err(format!("invalid dependency alias \"{alias}\" in metadata")) + } +} + +/// Source keys must stay inside their owning ingot directory. +fn checked_source_path(rel: &str) -> Result<(), String> { + let valid = !rel.is_empty() + && !rel.contains('\\') + && !Utf8Path::new(rel).is_absolute() + && rel + .split('/') + .all(|component| !component.is_empty() && component != "." && component != ".."); + if valid { + Ok(()) + } else { + Err(format!("invalid source path \"{rel}\" in metadata")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::tempdir; + + fn minimal_metadata(source_key: &str) -> Value { + serde_json::json!({ + "version": 1, + "language": "Fe", + "sources": { + source_key: { "content": "pub contract Foo {\n}\n" }, + }, + "settings": { + "ingots": [{ + "name": "app", + "version": "0.1.0", + "namespace": "", + "arithmetic": "checked", + "dependencies": {}, + }], + }, + }) + } + + #[test] + fn validate_rejects_wrong_language() { + let mut metadata = minimal_metadata("src/lib.fe"); + metadata["language"] = "Solidity".into(); + let err = validate_metadata(&metadata).unwrap_err(); + assert!(err.contains("language"), "unexpected error: {err}"); + } + + #[test] + fn validate_rejects_missing_sources() { + let mut metadata = minimal_metadata("src/lib.fe"); + metadata.as_object_mut().unwrap().remove("sources"); + let err = validate_metadata(&metadata).unwrap_err(); + assert!(err.contains("sources"), "unexpected error: {err}"); + } + + #[test] + fn reconstruct_rejects_escaping_source_paths() { + for key in ["../evil.fe", "/etc/evil.fe", "src/../../evil.fe"] { + let metadata = minimal_metadata(key); + let temp = tempdir().expect("tempdir"); + let dest = Utf8Path::from_path(temp.path()).expect("utf8 tempdir"); + let err = reconstruct_project(&metadata, dest) + .expect_err(&format!("key {key} must be rejected")); + assert!( + err.contains("invalid source path"), + "unexpected error: {err}" + ); + } + } + + #[test] + fn reconstruct_rejects_escaping_namespaces() { + let mut metadata = minimal_metadata("src/lib.fe"); + metadata["settings"]["ingots"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({ + "name": "dep", + "namespace": "../dep", + "dependencies": {}, + })); + let temp = tempdir().expect("tempdir"); + let dest = Utf8Path::from_path(temp.path()).expect("utf8 tempdir"); + let err = reconstruct_project(&metadata, dest).expect_err("namespace must be rejected"); + assert!( + err.contains("invalid ingot namespace"), + "unexpected error: {err}" + ); + } + + #[test] + fn reconstruct_accepts_url_encoded_standalone_root_name() { + // Standalone-target metadata records the parent directory's URL + // basename as the root name, which may contain characters (e.g. `%20`) + // that are not valid directory names. Reconstruction must sanitize, + // not reject, since the emitter produces such names. + let mut metadata = minimal_metadata("main.fe"); + metadata["settings"]["ingots"][0]["name"] = "my%20project".into(); + metadata["settings"]["ingots"][0]["version"] = Value::Null; + let temp = tempdir().expect("tempdir"); + let dest = Utf8Path::from_path(temp.path()).expect("utf8 tempdir"); + let root_dir = reconstruct_project(&metadata, dest).expect("reconstruct"); + assert_eq!(root_dir, dest.join("my_20project")); + let toml = fs::read_to_string(root_dir.join("fe.toml").as_std_path()).expect("fe.toml"); + assert!(toml.contains("name = \"my_20project\""), "toml: {toml}"); + assert!(root_dir.join("src/main.fe").is_file()); + } + + #[test] + fn reconstruct_root_dir_dodges_dependency_namespace_collision() { + let mut metadata = minimal_metadata("src/lib.fe"); + metadata["settings"]["ingots"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({ + "name": "app", + "version": "1.0.0", + "namespace": "app", + "arithmetic": "checked", + "dependencies": {}, + })); + metadata["sources"].as_object_mut().unwrap().insert( + "app/src/lib.fe".to_string(), + serde_json::json!({ "content": "pub fn helper() {}\n" }), + ); + let temp = tempdir().expect("tempdir"); + let dest = Utf8Path::from_path(temp.path()).expect("utf8 tempdir"); + let root_dir = reconstruct_project(&metadata, dest).expect("reconstruct"); + assert_eq!(root_dir, dest.join("app-root")); + assert!(root_dir.join("fe.toml").is_file()); + assert!(dest.join("app/fe.toml").is_file()); + } +} diff --git a/crates/fe/tests/cli_output.rs b/crates/fe/tests/cli_output.rs index 7fc19d16da..29fb3da33a 100644 --- a/crates/fe/tests/cli_output.rs +++ b/crates/fe/tests/cli_output.rs @@ -105,6 +105,33 @@ fn run_fe_main_in_dir(args: &[&str], cwd: &Path) -> (String, i32) { (out.combined(), out.exit_code) } +fn run_fe_main_with_stdin(args: &[&str], stdin_data: &str) -> (String, i32) { + use std::io::Write; + use std::process::Stdio; + + let mut child = Command::new(fe_binary()) + .args(args) + .env("NO_COLOR", "1") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap_or_else(|_| panic!("Failed to spawn fe {args:?}")); + child + .stdin + .take() + .expect("child stdin") + .write_all(stdin_data.as_bytes()) + .expect("write stdin"); + let output = child.wait_with_output().expect("wait for fe"); + let out = FeOutput { + stdout: normalize_output(&String::from_utf8_lossy(&output.stdout)), + stderr: normalize_output(&String::from_utf8_lossy(&output.stderr)), + exit_code: output.status.code().unwrap_or(-1), + }; + (out.combined(), out.exit_code) +} + #[test] fn test_cli_check_invalid_named_const_used_in_type_position_reports_error_instead_of_panicking() { let temp = tempdir().expect("tempdir"); @@ -909,22 +936,19 @@ fn test_cli_build_metadata_round_trip_reproduces_runtime_bytecode() { assert_eq!(exit_code, 0, "original build failed:\n{output}"); let original_runtime = fs::read_to_string(out_dir.join("Foo.runtime.bin")).expect("read original runtime.bin"); - let metadata: Value = serde_json::from_str( - &fs::read_to_string(out_dir.join("Foo.metadata.json")).expect("read metadata"), - ) - .expect("parse metadata"); - // Reconstruct a fresh project solely from the metadata, then rebuild. + // Rebuild solely from the metadata artifact via `--from-metadata`. + let metadata_path = out_dir.join("Foo.metadata.json"); let recon = tempdir().expect("recon tempdir"); - let root_dir = reconstruct_project_from_metadata(&metadata, recon.path()); let recon_out = recon.path().join("out"); let (output, exit_code) = run_fe_main(&[ "build", + "--from-metadata", + metadata_path.to_str().expect("metadata utf8"), "--emit", "runtime-bytecode", "--out-dir", recon_out.to_str().expect("out utf8"), - root_dir.to_str().expect("root utf8"), ]); assert_eq!(exit_code, 0, "rebuild from metadata failed:\n{output}"); let rebuilt_runtime = @@ -1135,19 +1159,20 @@ fn test_cli_build_emit_metadata_disambiguates_same_named_dependencies() { .collect(); assert!(all_content.contains("helper_one") && all_content.contains("helper_two")); - // Round-trip: reconstruct from metadata and rebuild to byte-identical runtime bytecode. + // Round-trip via `--from-metadata`: rebuild to byte-identical runtime bytecode. let original_runtime = fs::read_to_string(out_dir.join("Foo.runtime.bin")).expect("read original runtime.bin"); + let metadata_path = out_dir.join("Foo.metadata.json"); let recon = tempdir().expect("recon tempdir"); - let root_dir = reconstruct_project_from_metadata(&value, recon.path()); let recon_out = recon.path().join("out"); let (output, exit_code) = run_fe_main(&[ "build", + "--from-metadata", + metadata_path.to_str().expect("metadata utf8"), "--emit", "runtime-bytecode", "--out-dir", recon_out.to_str().expect("out utf8"), - root_dir.to_str().expect("root utf8"), ]); assert_eq!(exit_code, 0, "rebuild from metadata failed:\n{output}"); let rebuilt_runtime = @@ -1216,16 +1241,17 @@ fn test_cli_build_emit_metadata_disambiguates_dependency_from_root_source_path() let original_runtime = fs::read_to_string(out_dir.join("Foo.runtime.bin")).expect("read original runtime.bin"); + let metadata_path = out_dir.join("Foo.metadata.json"); let recon = tempdir().expect("recon tempdir"); - let root_dir = reconstruct_project_from_metadata(&metadata, recon.path()); let recon_out = recon.path().join("out"); let (output, exit_code) = run_fe_main(&[ "build", + "--from-metadata", + metadata_path.to_str().expect("metadata utf8"), "--emit", "runtime-bytecode", "--out-dir", recon_out.to_str().expect("out utf8"), - root_dir.to_str().expect("root utf8"), ]); assert_eq!(exit_code, 0, "rebuild from metadata failed:\n{output}"); let rebuilt_runtime = @@ -1319,19 +1345,21 @@ fn test_cli_build_emit_metadata_preserves_dependency_arithmetic_across_edge_type "internal edge must not be forced" ); - // Round-trip: the flattened reconstruction must still reproduce the exact bytecode. + // Round-trip via `--from-metadata`: the flattened reconstruction must still + // reproduce the exact bytecode. let original_runtime = fs::read_to_string(out_dir.join("App.runtime.bin")).expect("read original runtime.bin"); + let metadata_path = out_dir.join("App.metadata.json"); let recon = tempdir().expect("recon tempdir"); - let root_dir = reconstruct_project_from_metadata(&value, recon.path()); let recon_out = recon.path().join("out"); let (output, exit_code) = run_fe_main(&[ "build", + "--from-metadata", + metadata_path.to_str().expect("metadata utf8"), "--emit", "runtime-bytecode", "--out-dir", recon_out.to_str().expect("out utf8"), - root_dir.to_str().expect("root utf8"), ]); assert_eq!(exit_code, 0, "rebuild from metadata failed:\n{output}"); let rebuilt_runtime = @@ -1369,95 +1397,389 @@ fn write_app_with_path_dependency(root: &Path) { .expect("write mylib/src/lib.fe"); } -/// Reconstruct a buildable project under `dest` purely from a `metadata.json` value: one directory -/// per `settings.ingots[]` entry (root at `dest/`, deps at `dest/`), regenerating -/// each `fe.toml` and writing every `sources` entry into its namespaced layout. Returns the root -/// ingot directory. -fn reconstruct_project_from_metadata(metadata: &Value, dest: &Path) -> std::path::PathBuf { - let ingots = metadata["settings"]["ingots"] - .as_array() - .expect("ingots array"); +#[test] +fn test_cli_build_from_metadata_reads_stdin() { + let temp = tempdir().expect("tempdir"); + let root = temp.path(); + write_app_with_path_dependency(root); - let dir_for = |namespace: &str, name: &str| -> std::path::PathBuf { - if namespace.is_empty() { - dest.join(name) - } else { - dest.join(namespace) - } - }; + let out_dir = root.join("app/out"); + let (output, exit_code) = run_fe_main(&[ + "build", + "--emit", + "metadata,runtime-bytecode", + "--out-dir", + out_dir.to_str().expect("out utf8"), + root.join("app").to_str().expect("app utf8"), + ]); + assert_eq!(exit_code, 0, "original build failed:\n{output}"); + let original_runtime = + fs::read_to_string(out_dir.join("Foo.runtime.bin")).expect("read original runtime.bin"); + let metadata = fs::read_to_string(out_dir.join("Foo.metadata.json")).expect("read metadata"); - // Non-root namespaces, used to route each source key to its owning ingot. - let non_root_namespaces: Vec = ingots - .iter() - .filter_map(|i| { - let ns = i["namespace"].as_str().unwrap_or(""); - (!ns.is_empty()).then(|| ns.to_string()) - }) - .collect(); + // `--from-metadata -` reads the JSON from stdin. + let recon = tempdir().expect("recon tempdir"); + let recon_out = recon.path().join("out"); + let (output, exit_code) = run_fe_main_with_stdin( + &[ + "build", + "--from-metadata", + "-", + "--emit", + "runtime-bytecode", + "--out-dir", + recon_out.to_str().expect("out utf8"), + ], + &metadata, + ); + assert_eq!( + exit_code, 0, + "rebuild from stdin metadata failed:\n{output}" + ); + let rebuilt_runtime = + fs::read_to_string(recon_out.join("Foo.runtime.bin")).expect("read rebuilt runtime.bin"); + assert_eq!( + original_runtime, rebuilt_runtime, + "runtime bytecode rebuilt from stdin metadata must be byte-identical" + ); +} - let mut root_dir = dest.to_path_buf(); +#[test] +fn test_cli_build_from_metadata_round_trips_standalone_file() { + // Standalone-target metadata keys its single source by file basename and + // records `version: null` for the pseudo-ingot; the reconstruction must + // still produce a loadable project (source under `src/`, valid fe.toml). + let temp = tempdir().expect("tempdir"); + let file = temp.path().join("counter.fe"); + fs::write( + &file, + "pub msg FooMsg {\n #[selector = sol(\"run()\")]\n Run -> u256,\n}\n\npub contract Foo {\n recv FooMsg {\n Run -> u256 {\n 42\n }\n }\n}\n", + ) + .expect("write counter.fe"); - // 1. Regenerate each ingot's fe.toml. - for ingot in ingots { - let name = ingot["name"].as_str().expect("ingot name"); - let namespace = ingot["namespace"].as_str().unwrap_or(""); - let dir = dir_for(namespace, name); - fs::create_dir_all(dir.join("src")).expect("create ingot src"); - if namespace.is_empty() { - root_dir = dir.clone(); - } + let out_dir = temp.path().join("out"); + let (output, exit_code) = run_fe_main(&[ + "build", + "--emit", + "metadata,runtime-bytecode", + "--out-dir", + out_dir.to_str().expect("out utf8"), + file.to_str().expect("file utf8"), + ]); + assert_eq!(exit_code, 0, "original standalone build failed:\n{output}"); + let original_runtime = + fs::read_to_string(out_dir.join("Foo.runtime.bin")).expect("read original runtime.bin"); - let mut toml = format!("[ingot]\nname = \"{name}\"\n"); - if let Some(version) = ingot["version"].as_str() { - toml.push_str(&format!("version = \"{version}\"\n")); - } - if let Some(arith) = ingot["arithmetic"].as_str() { - toml.push_str(&format!("arithmetic = \"{arith}\"\n")); - } - let deps = ingot["dependencies"].as_object().expect("dependencies"); - let path_deps: Vec<(&String, &str)> = deps - .iter() - .filter_map(|(alias, target)| { - let target = target.as_str()?; - // std/core are provided by the compiler; only scaffold real path deps. - (target != "std" && target != "core").then_some((alias, target)) - }) - .collect(); - if !path_deps.is_empty() { - toml.push_str("\n[dependencies]\n"); - for (alias, target) in path_deps { - toml.push_str(&format!("{alias} = {{ path = \"../{target}\" }}\n")); - } - } - fs::write(dir.join("fe.toml"), toml).expect("write fe.toml"); - } + let metadata_path = out_dir.join("Foo.metadata.json"); + let recon = tempdir().expect("recon tempdir"); + let recon_out = recon.path().join("out"); + let (output, exit_code) = run_fe_main(&[ + "build", + "--from-metadata", + metadata_path.to_str().expect("metadata utf8"), + "--emit", + "runtime-bytecode", + "--out-dir", + recon_out.to_str().expect("out utf8"), + ]); + assert_eq!( + exit_code, 0, + "rebuild from standalone metadata failed:\n{output}" + ); + let rebuilt_runtime = + fs::read_to_string(recon_out.join("Foo.runtime.bin")).expect("read rebuilt runtime.bin"); + assert_eq!( + original_runtime, rebuilt_runtime, + "runtime bytecode rebuilt from standalone metadata must be byte-identical" + ); +} - // 2. Write every source into its owning ingot's namespaced src/ layout. - let sources = metadata["sources"].as_object().expect("sources object"); - for (key, source) in sources { - let content = source["content"].as_str().unwrap_or(""); - let (namespace, rel) = non_root_namespaces - .iter() - .find_map(|ns| { - key.strip_prefix(&format!("{ns}/")) - .map(|rel| (ns.as_str(), rel)) - }) - .unwrap_or(("", key.as_str())); - - // Find the owning ingot's directory (by namespace) to resolve its name for the root case. - let dir = ingots - .iter() - .find(|i| i["namespace"].as_str().unwrap_or("") == namespace) - .map(|i| dir_for(namespace, i["name"].as_str().unwrap_or("dep"))) - .expect("owning ingot for source"); - let target = dir.join(rel); - if let Some(parent) = target.parent() { - fs::create_dir_all(parent).expect("create source parent"); - } - fs::write(target, content).expect("write source"); +#[test] +fn test_cli_build_from_metadata_defaults_to_compilation_target_contract() { + let temp = tempdir().expect("tempdir"); + let src_dir = temp.path().join("src"); + fs::create_dir_all(&src_dir).expect("create src dir"); + fs::write( + temp.path().join("fe.toml"), + "[ingot]\nname = \"two_contracts\"\nversion = \"0.1.0\"\n", + ) + .expect("write fe.toml"); + fs::write( + src_dir.join("lib.fe"), + "pub contract Foo {\n}\n\npub contract Bar {\n}\n", + ) + .expect("write lib.fe"); + + let out_dir = temp.path().join("out"); + let (output, exit_code) = run_fe_main(&[ + "build", + "--emit", + "metadata", + "--out-dir", + out_dir.to_str().expect("out utf8"), + temp.path().to_str().expect("project utf8"), + ]); + assert_eq!(exit_code, 0, "original build failed:\n{output}"); + + // Foo's metadata records `compilationTarget: Foo`; the rebuild must only + // produce Foo's artifacts even though the sources define Bar as well. + let metadata_path = out_dir.join("Foo.metadata.json"); + let recon = tempdir().expect("recon tempdir"); + let recon_out = recon.path().join("out"); + let (output, exit_code) = run_fe_main(&[ + "build", + "--from-metadata", + metadata_path.to_str().expect("metadata utf8"), + "--emit", + "runtime-bytecode", + "--out-dir", + recon_out.to_str().expect("out utf8"), + ]); + assert_eq!(exit_code, 0, "rebuild from metadata failed:\n{output}"); + assert!( + recon_out.join("Foo.runtime.bin").is_file(), + "missing Foo artifact:\n{output}" + ); + assert!( + !recon_out.join("Bar.runtime.bin").exists(), + "Bar must not be built by default:\n{output}" + ); + + // `--contract` overrides the recorded compilation target. + let recon_out_bar = recon.path().join("out-bar"); + let (output, exit_code) = run_fe_main(&[ + "build", + "--from-metadata", + metadata_path.to_str().expect("metadata utf8"), + "--contract", + "Bar", + "--emit", + "runtime-bytecode", + "--out-dir", + recon_out_bar.to_str().expect("out utf8"), + ]); + assert_eq!(exit_code, 0, "rebuild with --contract failed:\n{output}"); + assert!( + recon_out_bar.join("Bar.runtime.bin").is_file(), + "missing Bar artifact:\n{output}" + ); +} + +#[test] +fn test_cli_build_from_metadata_optimize_override_warns() { + let temp = tempdir().expect("tempdir"); + let root = temp.path(); + write_app_with_path_dependency(root); + + let out_dir = root.join("app/out"); + let (output, exit_code) = run_fe_main(&[ + "build", + "--emit", + "metadata", + "--out-dir", + out_dir.to_str().expect("out utf8"), + root.join("app").to_str().expect("app utf8"), + ]); + assert_eq!(exit_code, 0, "original build failed:\n{output}"); + let metadata_path = out_dir.join("Foo.metadata.json"); + + // The metadata records level `1` (the default); an explicit `-O 0` wins, + // with a warning that the rebuilt bytecode deviates. + let recon = tempdir().expect("recon tempdir"); + let recon_out = recon.path().join("out"); + let (output, exit_code) = run_fe_main(&[ + "build", + "--from-metadata", + metadata_path.to_str().expect("metadata utf8"), + "-O", + "0", + "--emit", + "runtime-bytecode", + "--out-dir", + recon_out.to_str().expect("out utf8"), + ]); + assert_eq!(exit_code, 0, "rebuild with -O override failed:\n{output}"); + assert!( + output.contains("Warning: -O 0 overrides optimizer level 1"), + "expected override warning:\n{output}" + ); + assert!( + recon_out.join("Foo.runtime.bin").is_file(), + "missing rebuilt artifact:\n{output}" + ); +} + +#[test] +fn test_cli_build_from_metadata_warns_on_compiler_version_mismatch() { + let temp = tempdir().expect("tempdir"); + let root = temp.path(); + write_app_with_path_dependency(root); + + let out_dir = root.join("app/out"); + let (output, exit_code) = run_fe_main(&[ + "build", + "--emit", + "metadata", + "--out-dir", + out_dir.to_str().expect("out utf8"), + root.join("app").to_str().expect("app utf8"), + ]); + assert_eq!(exit_code, 0, "original build failed:\n{output}"); + + let mut metadata: Value = serde_json::from_str( + &fs::read_to_string(out_dir.join("Foo.metadata.json")).expect("read metadata"), + ) + .expect("parse metadata"); + metadata["compiler"]["version"] = "0.0.1".into(); + let edited_path = root.join("edited.metadata.json"); + fs::write(&edited_path, metadata.to_string()).expect("write edited metadata"); + + let recon = tempdir().expect("recon tempdir"); + let recon_out = recon.path().join("out"); + let (output, exit_code) = run_fe_main(&[ + "build", + "--from-metadata", + edited_path.to_str().expect("metadata utf8"), + "--emit", + "runtime-bytecode", + "--out-dir", + recon_out.to_str().expect("out utf8"), + ]); + assert_eq!(exit_code, 0, "version mismatch must not abort:\n{output}"); + assert!( + output.contains("Warning: metadata was produced by fe 0.0.1"), + "expected version mismatch warning:\n{output}" + ); +} + +#[test] +fn test_cli_build_from_metadata_warns_on_compiler_commit_mismatch() { + let temp = tempdir().expect("tempdir"); + let root = temp.path(); + write_app_with_path_dependency(root); + + let out_dir = root.join("app/out"); + let (output, exit_code) = run_fe_main(&[ + "build", + "--emit", + "metadata", + "--out-dir", + out_dir.to_str().expect("out utf8"), + root.join("app").to_str().expect("app utf8"), + ]); + assert_eq!(exit_code, 0, "original build failed:\n{output}"); + + let mut metadata: Value = serde_json::from_str( + &fs::read_to_string(out_dir.join("Foo.metadata.json")).expect("read metadata"), + ) + .expect("parse metadata"); + if metadata["compiler"]["commit"].as_str().is_none() { + // The test binary was built without git information, so the rebuild + // has no commit to compare against. + return; } + metadata["compiler"]["commit"] = "deadbee".into(); + let edited_path = root.join("edited.metadata.json"); + fs::write(&edited_path, metadata.to_string()).expect("write edited metadata"); + + let recon = tempdir().expect("recon tempdir"); + let recon_out = recon.path().join("out"); + let (output, exit_code) = run_fe_main(&[ + "build", + "--from-metadata", + edited_path.to_str().expect("metadata utf8"), + "--emit", + "runtime-bytecode", + "--out-dir", + recon_out.to_str().expect("out utf8"), + ]); + assert_eq!(exit_code, 0, "commit mismatch must not abort:\n{output}"); + assert!( + output.contains("Warning: metadata was produced by fe commit deadbee"), + "expected commit mismatch warning:\n{output}" + ); +} + +#[test] +fn test_cli_build_from_metadata_rejects_invalid_input() { + let temp = tempdir().expect("tempdir"); + + // Broken JSON. + let broken = temp.path().join("broken.metadata.json"); + fs::write(&broken, "{ not json").expect("write broken metadata"); + let (output, exit_code) = run_fe_main(&[ + "build", + "--from-metadata", + broken.to_str().expect("path utf8"), + ]); + assert_eq!(exit_code, 1, "expected exit 1:\n{output}"); + assert!( + output.contains("Failed to parse metadata JSON"), + "expected JSON parse error:\n{output}" + ); + + // Missing `sources`. + let no_sources = temp.path().join("no_sources.metadata.json"); + fs::write( + &no_sources, + r#"{"version":1,"language":"Fe","settings":{"ingots":[]}}"#, + ) + .expect("write metadata without sources"); + let (output, exit_code) = run_fe_main(&[ + "build", + "--from-metadata", + no_sources.to_str().expect("path utf8"), + ]); + assert_eq!(exit_code, 1, "expected exit 1:\n{output}"); + assert!( + output.contains("missing the `sources` object"), + "expected missing-sources error:\n{output}" + ); - root_dir + // Wrong language. + let wrong_language = temp.path().join("wrong_language.metadata.json"); + fs::write( + &wrong_language, + r#"{"version":1,"language":"Solidity","sources":{"a.sol":{"content":""}},"settings":{"ingots":[]}}"#, + ) + .expect("write non-Fe metadata"); + let (output, exit_code) = run_fe_main(&[ + "build", + "--from-metadata", + wrong_language.to_str().expect("path utf8"), + ]); + assert_eq!(exit_code, 1, "expected exit 1:\n{output}"); + assert!( + output.contains("`language` must be \"Fe\""), + "expected language error:\n{output}" + ); +} + +#[test] +fn test_cli_build_from_metadata_conflicts_with_path_and_standalone() { + let (output, exit_code) = run_fe_main(&[ + "build", + "--from-metadata", + "Foo.metadata.json", + "some/project/path", + ]); + assert_eq!(exit_code, 2, "expected clap error exit 2:\n{output}"); + assert!( + output.contains("cannot be used with"), + "expected conflict error:\n{output}" + ); + + let (output, exit_code) = run_fe_main(&[ + "build", + "--from-metadata", + "Foo.metadata.json", + "--standalone", + ]); + assert_eq!(exit_code, 2, "expected clap error exit 2:\n{output}"); + assert!( + output.contains("cannot be used with"), + "expected conflict error:\n{output}" + ); } #[test] diff --git a/newsfragments/1505.feature.md b/newsfragments/1505.feature.md new file mode 100644 index 0000000000..73d98e10b1 --- /dev/null +++ b/newsfragments/1505.feature.md @@ -0,0 +1 @@ +Added `fe build --from-metadata ` to rebuild a contract from a `.metadata.json` recompilation input (as produced by `fe build --emit metadata`); `-` reads the JSON from stdin. The recorded project is materialized into a temporary directory and built with the settings captured in the metadata: the contract from `settings.compilationTarget` (overridable with `--contract`) and the recorded optimizer level (an explicit `-O` wins, with a warning that the rebuilt bytecode no longer matches the verified artifact). A compiler version mismatch warns on stderr without aborting. Artifacts are written to `--out-dir`, defaulting to `./out`. This lets external toolchains such as Foundry or source verifiers recompile Fe contracts from a single self-contained JSON document.