From 4a5a68466b10d5eac453cfb877bea86872d69dfc Mon Sep 17 00:00:00 2001 From: Charlie <5764343+charlielye@users.noreply.github.com> Date: Tue, 23 Jun 2026 18:27:15 +0000 Subject: [PATCH] refactor: cut TS world state and NAPI AVM over to WSDB IPC; delete NAPI WSDB --- aztec-up/bootstrap.sh | 11 +- .../barretenberg/nodejs_module/CMakeLists.txt | 2 +- .../avm_simulate/avm_simulate_napi.cpp | 30 +- .../avm_simulate/avm_simulate_napi.hpp | 2 +- .../nodejs_module/init_module.cpp | 2 - .../nodejs_module/world_state/world_state.cpp | 990 ------------------ .../nodejs_module/world_state/world_state.hpp | 84 -- .../world_state/world_state_message.hpp | 272 ----- .../cpp/src/barretenberg/vm2/avm_sim_api.cpp | 36 +- .../cpp/src/barretenberg/vm2/avm_sim_api.hpp | 2 +- .../barretenberg/vm2/simulation_helper.cpp | 34 +- .../barretenberg/vm2/simulation_helper.hpp | 18 +- docs/examples/bootstrap.sh | 19 +- ipc-codegen/src/typescript_package_codegen.ts | 48 +- ipc-runtime/cpp/ipc_runtime/ipc_server.hpp | 21 +- ipc-runtime/cpp/ipc_runtime/shm.test.cpp | 221 +++- ipc-runtime/cpp/ipc_runtime/shm/spsc_shm.cpp | 13 +- release-image/Dockerfile.base.dockerignore | 3 + release-image/Dockerfile.dockerignore | 6 + release-image/bootstrap.sh | 4 +- .../e2e_l1_publisher/e2e_l1_publisher.test.ts | 2 +- yarn-project/native/src/native_module.ts | 10 +- .../tx_validator/tx_validator_bench.test.ts | 4 +- yarn-project/package.json | 6 + ...ghtweight_checkpoint_builder.bench.test.ts | 2 +- .../lightweight_checkpoint_builder.test.ts | 2 +- .../prover-client/src/mocks/test_context.ts | 3 +- .../src/actions/rerun-epoch-proving-job.ts | 2 +- .../src/public/hinting_db_sources.ts | 4 + .../public_processor/guarded_merkle_tree.ts | 7 +- .../cpp_public_tx_simulator.ts | 15 +- .../cpp_vs_ts_public_tx_simulator.ts | 13 +- .../src/interfaces/merkle_tree_operations.ts | 11 +- .../src/world-state/world_state_revision.ts | 33 - .../src/validator.integration.test.ts | 9 +- yarn-project/world-state/package.json | 2 + .../src/instrumentation/instrumentation.ts | 24 +- .../src/native/ipc_world_state_instance.ts | 830 +++++++++++++++ .../src/native/merkle_trees_facade.ts | 163 ++- .../world-state/src/native/message.ts | 309 +----- .../multi_connection_read.bench.test.ts | 116 ++ .../src/native/native_world_state.test.ts | 16 +- .../src/native/native_world_state.ts | 274 +++-- .../src/native/native_world_state_instance.ts | 386 ++----- .../src/native/world_state_operation.ts | 33 + .../src/native/world_state_ops_queue.ts | 190 ---- .../world-state/src/synchronizer/factory.ts | 1 - .../world-state/src/test/integration.test.ts | 2 +- yarn-project/world-state/src/testing.ts | 2 +- yarn-project/yarn.lock | 61 ++ 50 files changed, 1809 insertions(+), 2541 deletions(-) delete mode 100644 barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.cpp delete mode 100644 barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.hpp delete mode 100644 barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state_message.hpp create mode 100644 yarn-project/world-state/src/native/ipc_world_state_instance.ts create mode 100644 yarn-project/world-state/src/native/multi_connection_read.bench.test.ts create mode 100644 yarn-project/world-state/src/native/world_state_operation.ts delete mode 100644 yarn-project/world-state/src/native/world_state_ops_queue.ts diff --git a/aztec-up/bootstrap.sh b/aztec-up/bootstrap.sh index 4618253bca97..1bcfcca9bf49 100755 --- a/aztec-up/bootstrap.sh +++ b/aztec-up/bootstrap.sh @@ -1,11 +1,18 @@ #!/usr/bin/env bash source $(git rev-parse --show-toplevel)/ci3/source_bootstrap -hash=$(hash_str $(cache_content_hash ^aztec-up/) $(../yarn-project/bootstrap.sh hash)) +hash=$(hash_str $(cache_content_hash ^aztec-up/) $(../ipc-runtime/bootstrap.sh hash) $(../wsdb/bootstrap.sh hash) $(../yarn-project/bootstrap.sh hash)) # Bare aliases ("nightly", "latest") resolve to this major version. DEFAULT_MAJOR_VERSION=${AZTEC_TOOLCHAIN_DEFAULT_MAJOR_VERSION:-4} +function wsdb_package_dirs { + for package_dir in "$root"/wsdb/ts/packages/*; do + [ -d "$package_dir" ] && echo "$package_dir" + done + echo "$root/wsdb/ts" +} + function build { # Noop if user doesn't have docker. if ! command -v docker &>/dev/null; then @@ -103,7 +110,9 @@ EOF # TODO(AD): we have kludged a retry here. a local NPM install ought to be robust enough not to. echo "Deploying packages to local npm registry (version: $version)..." { + echo $root/ipc-runtime/ts echo $root/barretenberg/ts + wsdb_package_dirs $root/noir/bootstrap.sh get_projects $root/yarn-project/bootstrap.sh get_projects } | DRY_RUN= parallel --tag --line-buffer --halt now,fail=1 "retry 'cd {} && dump_fail \"deploy_npm $version\" >/dev/null'" diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/CMakeLists.txt b/barretenberg/cpp/src/barretenberg/nodejs_module/CMakeLists.txt index 1eb6cee25bd1..b3c83492445b 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/CMakeLists.txt +++ b/barretenberg/cpp/src/barretenberg/nodejs_module/CMakeLists.txt @@ -27,7 +27,7 @@ string(REGEX REPLACE "[\r\n\"]" "" NODE_API_HEADERS_DIR ${NODE_API_HEADERS_DIR}) add_library(nodejs_module SHARED ${SOURCE_FILES}) set_target_properties(nodejs_module PROPERTIES PREFIX "" SUFFIX ".node") target_include_directories(nodejs_module PRIVATE ${NODE_API_HEADERS_DIR} ${NODE_ADDON_API_DIR}) -target_link_libraries(nodejs_module PRIVATE world_state ipc vm2_sim) +target_link_libraries(nodejs_module PRIVATE ipc ipc_runtime vm2_sim wsdb_ipc_merkle_db) # On macOS, Node.js N-API symbols are provided by the runtime, not at link time if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/avm_simulate/avm_simulate_napi.cpp b/barretenberg/cpp/src/barretenberg/nodejs_module/avm_simulate/avm_simulate_napi.cpp index 5819bcf3a743..819dbf0b7f14 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/avm_simulate/avm_simulate_napi.cpp +++ b/barretenberg/cpp/src/barretenberg/nodejs_module/avm_simulate/avm_simulate_napi.cpp @@ -12,6 +12,8 @@ #include "barretenberg/vm2/avm_sim_api.hpp" #include "barretenberg/vm2/common/avm_io.hpp" #include "barretenberg/vm2/simulation/lib/cancellation_token.hpp" +#include "barretenberg/vm2_wsdb/wsdb_ipc_merkle_db.hpp" +#include "barretenberg/wsdb/generated/wsdb_ipc_client.hpp" namespace bb::nodejs { namespace { @@ -227,15 +229,13 @@ Napi::Value AvmSimulateNapi::simulate(const Napi::CallbackInfo& cb_info) env, ContractCallbacks::get(contract_provider, CALLBACK_REVERT_CHECKPOINT), CALLBACK_REVERT_CHECKPOINT), }; - /***************************** - *** WorldState (required) *** - *****************************/ - if (!cb_info[2].IsExternal()) { - throw Napi::TypeError::New(env, "Third argument must be a WorldState handle (External)"); + /********************************** + *** WSDB IPC path (required) *** + **********************************/ + if (!cb_info[2].IsString()) { + throw Napi::TypeError::New(env, "Third argument must be a WSDB IPC path (string)"); } - // Extract WorldState handle (3rd argument) - auto external = cb_info[2].As>(); - world_state::WorldState* ws_ptr = external.Data(); + std::string wsdb_ipc_path = cb_info[2].As().Utf8Value(); /*************************** *** LogLevel (optional) *** @@ -281,10 +281,8 @@ Napi::Value AvmSimulateNapi::simulate(const Napi::CallbackInfo& cb_info) **********************************************************/ auto deferred = std::make_shared(env); - // Run on a dedicated std::thread (not libuv pool) to prevent libuv thread pool - // exhaustion when callbacks need libuv threads for I/O. ThreadedAsyncOperation::Run( - env, deferred, [data, tsfns, logger_tsfn, ws_ptr, cancellation_token](msgpack::sbuffer& result_buffer) { + env, deferred, [data, tsfns, logger_tsfn, wsdb_ipc_path, cancellation_token](msgpack::sbuffer& result_buffer) { // Collect all thread-safe functions including logger for cleanup auto all_tsfns = tsfns.to_vector(); all_tsfns.push_back(logger_tsfn); @@ -309,10 +307,14 @@ Napi::Value AvmSimulateNapi::simulate(const Napi::CallbackInfo& cb_info) *tsfns.commit_checkpoint, *tsfns.revert_checkpoint); - // Create AVM API and run simulation with the callback-based contracts DB, - // WorldState reference, and optional cancellation token + // Connect to aztec-wsdb and wrap in a WsdbIpcMerkleDB that implements + // LowLevelMerkleDBInterface. The connection is per-simulation; aztec-wsdb is a + // long-running server that the TS layer spawned and owns. + bb::wsdb::WsdbIpcClient wsdb_client(wsdb_ipc_path); + bb::avm2::simulation::WsdbIpcMerkleDB merkle_db(wsdb_client, inputs.ws_revision); + avm2::AvmSimAPI avm; - avm2::TxSimulationResult result = avm.simulate(inputs, contract_db, *ws_ptr, cancellation_token); + avm2::TxSimulationResult result = avm.simulate(inputs, contract_db, merkle_db, cancellation_token); // Serialize the simulation result with msgpack into the return buffer to TS. msgpack::pack(result_buffer, result); diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/avm_simulate/avm_simulate_napi.hpp b/barretenberg/cpp/src/barretenberg/nodejs_module/avm_simulate/avm_simulate_napi.hpp index f24c598cba5f..b01ccc446a47 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/avm_simulate/avm_simulate_napi.hpp +++ b/barretenberg/cpp/src/barretenberg/nodejs_module/avm_simulate/avm_simulate_napi.hpp @@ -25,7 +25,7 @@ class AvmSimulateNapi { * - info[1]: Object with contract provider callbacks: * - getContractInstance(address: string): Promise * - getContractClass(classId: string): Promise - * - info[2]: External WorldState handle (pointer to world_state::WorldState) + * - info[2]: WSDB IPC path (string) — TS layer spawned aztec-wsdb at this path * - info[3]: Log level number (0-7) * - info[4]: External CancellationToken handle (optional) * diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/init_module.cpp b/barretenberg/cpp/src/barretenberg/nodejs_module/init_module.cpp index f3c7a3255ae5..5be9e9149a85 100644 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/init_module.cpp +++ b/barretenberg/cpp/src/barretenberg/nodejs_module/init_module.cpp @@ -2,12 +2,10 @@ #include "barretenberg/nodejs_module/lmdb_store/lmdb_store_wrapper.hpp" #include "barretenberg/nodejs_module/msgpack_client/msgpack_client_async.hpp" #include "barretenberg/nodejs_module/msgpack_client/msgpack_client_wrapper.hpp" -#include "barretenberg/nodejs_module/world_state/world_state.hpp" #include "napi.h" Napi::Object Init(Napi::Env env, Napi::Object exports) { - exports.Set(Napi::String::New(env, "WorldState"), bb::nodejs::WorldStateWrapper::get_class(env)); exports.Set(Napi::String::New(env, "LMDBStore"), bb::nodejs::lmdb_store::LMDBStoreWrapper::get_class(env)); exports.Set(Napi::String::New(env, "MsgpackClient"), bb::nodejs::msgpack_client::MsgpackClientWrapper::get_class(env)); diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.cpp b/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.cpp deleted file mode 100644 index cf55d7c5f6ec..000000000000 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.cpp +++ /dev/null @@ -1,990 +0,0 @@ -#include "barretenberg/world_state/world_state.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/response.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/messaging/header.hpp" -#include "barretenberg/nodejs_module/util/async_op.hpp" -#include "barretenberg/nodejs_module/world_state/world_state.hpp" -#include "barretenberg/nodejs_module/world_state/world_state_message.hpp" -#include "barretenberg/serialize/msgpack.hpp" -#include "barretenberg/world_state/fork.hpp" -#include "barretenberg/world_state/types.hpp" -#include "napi.h" - -using namespace bb::nodejs; -using namespace bb::world_state; -using namespace bb::crypto::merkle_tree; -using namespace bb::messaging; - -const uint64_t DEFAULT_MAP_SIZE = 1024UL * 1024; - -WorldStateWrapper::WorldStateWrapper(const Napi::CallbackInfo& info) - : ObjectWrap(info) -{ - uint64_t thread_pool_size = 16; - std::string data_dir; - std::unordered_map map_size{ - { MerkleTreeId::ARCHIVE, DEFAULT_MAP_SIZE }, - { MerkleTreeId::NULLIFIER_TREE, DEFAULT_MAP_SIZE }, - { MerkleTreeId::NOTE_HASH_TREE, DEFAULT_MAP_SIZE }, - { MerkleTreeId::PUBLIC_DATA_TREE, DEFAULT_MAP_SIZE }, - { MerkleTreeId::L1_TO_L2_MESSAGE_TREE, DEFAULT_MAP_SIZE }, - }; - std::unordered_map tree_height; - std::unordered_map tree_prefill; - std::vector prefilled_public_data; - std::vector tree_ids{ - MerkleTreeId::NULLIFIER_TREE, MerkleTreeId::NOTE_HASH_TREE, MerkleTreeId::PUBLIC_DATA_TREE, - MerkleTreeId::L1_TO_L2_MESSAGE_TREE, MerkleTreeId::ARCHIVE, - }; - uint32_t initial_header_generator_point = 0; - - Napi::Env env = info.Env(); - - size_t data_dir_index = 0; - if (info.Length() > data_dir_index && info[data_dir_index].IsString()) { - data_dir = info[data_dir_index].As(); - } else { - throw Napi::TypeError::New(env, "Directory needs to be a string"); - } - - size_t tree_height_index = 1; - if (info.Length() > tree_height_index && info[tree_height_index].IsObject()) { - Napi::Object obj = info[tree_height_index].As(); - - for (auto tree_id : tree_ids) { - if (obj.Has(tree_id)) { - tree_height[tree_id] = obj.Get(tree_id).As().Uint32Value(); - } - } - } else { - throw Napi::TypeError::New(env, "Tree heights must be a map"); - } - - size_t tree_prefill_index = 2; - if (info.Length() > tree_prefill_index && info[tree_prefill_index].IsObject()) { - Napi::Object obj = info[tree_prefill_index].As(); - - for (auto tree_id : tree_ids) { - if (obj.Has(tree_id)) { - tree_prefill[tree_id] = obj.Get(tree_id).As().Uint32Value(); - } - } - } else { - throw Napi::TypeError::New(env, "Tree prefill must be a map"); - } - - size_t prefilled_public_data_index = 3; - if (info.Length() > prefilled_public_data_index && info[prefilled_public_data_index].IsArray()) { - Napi::Array arr = info[prefilled_public_data_index].As(); - for (uint32_t i = 0; i < arr.Length(); ++i) { - Napi::Array deserialized = arr.Get(i).As(); - if (deserialized.Length() != 2 || !deserialized.Get(uint32_t(0)).IsBuffer() || - !deserialized.Get(uint32_t(1)).IsBuffer()) { - throw Napi::TypeError::New(env, "Prefilled public data value must be a buffer array of size 2"); - } - Napi::Buffer slot_buf = deserialized.Get(uint32_t(0)).As>(); - Napi::Buffer value_buf = deserialized.Get(uint32_t(1)).As>(); - uint256_t slot = 0; - uint256_t value = 0; - for (size_t j = 0; j < 32; ++j) { - slot = (slot << 8) | slot_buf[j]; - value = (value << 8) | value_buf[j]; - } - prefilled_public_data.push_back(PublicDataLeafValue(slot, value)); - } - } else { - throw Napi::TypeError::New(env, "Prefilled public data must be an array"); - } - - size_t initial_header_generator_point_index = 4; - if (info.Length() > initial_header_generator_point_index && info[initial_header_generator_point_index].IsNumber()) { - initial_header_generator_point = info[initial_header_generator_point_index].As().Uint32Value(); - } else { - throw Napi::TypeError::New(env, "Header generator point needs to be a number"); - } - - uint64_t genesis_timestamp = 0; - size_t genesis_timestamp_index = 5; - if (info.Length() > genesis_timestamp_index) { - if (info[genesis_timestamp_index].IsNumber()) { - genesis_timestamp = static_cast(info[genesis_timestamp_index].As().Int64Value()); - } else { - throw Napi::TypeError::New(env, "Genesis timestamp needs to be a number"); - } - } - - // optional parameters - size_t map_size_index = 6; - if (info.Length() > map_size_index) { - if (info[map_size_index].IsObject()) { - Napi::Object obj = info[map_size_index].As(); - - for (auto tree_id : tree_ids) { - if (obj.Has(tree_id)) { - // Int64Value is the widest integer accessor in N-API (no Uint64Value exists) - int64_t val = obj.Get(tree_id).As().Int64Value(); - if (val <= 0) { - throw Napi::TypeError::New(env, "Map size must be a positive number"); - } - map_size[tree_id] = static_cast(val); - } - } - } else if (info[map_size_index].IsNumber()) { - // Int64Value is the widest integer accessor in N-API (no Uint64Value exists) - int64_t val = info[map_size_index].As().Int64Value(); - if (val <= 0) { - throw Napi::TypeError::New(env, "Map size must be a positive number"); - } - uint64_t size = static_cast(val); - for (auto tree_id : tree_ids) { - map_size[tree_id] = size; - } - } else { - throw Napi::TypeError::New(env, "Map size must be a number or an object"); - } - } - - size_t thread_pool_size_index = 7; - if (info.Length() > thread_pool_size_index) { - if (!info[thread_pool_size_index].IsNumber()) { - throw Napi::TypeError::New(env, "Thread pool size must be a number"); - } - - thread_pool_size = info[thread_pool_size_index].As().Uint32Value(); - } - - // `ephemeral` opens each underlying LMDB env with `MDB_NOSYNC | MDB_NOMETASYNC` — - // commits never block on fsync, files stay sparse, and a crash mid-write yields an - // unrecoverable env. Intended for throwaway scratch state (TXE test sessions). - bool ephemeral = false; - size_t ephemeral_index = 8; - if (info.Length() > ephemeral_index) { - if (!info[ephemeral_index].IsBoolean()) { - throw Napi::TypeError::New(env, "Ephemeral flag must be a boolean"); - } - ephemeral = info[ephemeral_index].As().Value(); - } - - _ws = std::make_unique(thread_pool_size, - data_dir, - map_size, - tree_height, - tree_prefill, - prefilled_public_data, - initial_header_generator_point, - genesis_timestamp, - ephemeral); - - _dispatcher.register_target( - WorldStateMessageType::GET_TREE_INFO, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return get_tree_info(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::GET_STATE_REFERENCE, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return get_state_reference(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::GET_INITIAL_STATE_REFERENCE, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return get_initial_state_reference(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::GET_LEAF_VALUE, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return get_leaf_value(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::GET_LEAF_PREIMAGE, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return get_leaf_preimage(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::GET_SIBLING_PATH, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return get_sibling_path(obj, buffer); }); - - _dispatcher.register_target(WorldStateMessageType::GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { - return get_block_numbers_for_leaf_indices(obj, buffer); - }); - - _dispatcher.register_target( - WorldStateMessageType::FIND_LEAF_INDICES, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return find_leaf_indices(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::FIND_SIBLING_PATHS, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return find_sibling_paths(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::FIND_LOW_LEAF, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return find_low_leaf(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::APPEND_LEAVES, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return append_leaves(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::BATCH_INSERT, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return batch_insert(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::SEQUENTIAL_INSERT, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return sequential_insert(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::UPDATE_ARCHIVE, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return update_archive(obj, buffer); }); - - _dispatcher.register_target(WorldStateMessageType::COMMIT, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return commit(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::ROLLBACK, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return rollback(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::SYNC_BLOCK, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return sync_block(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::CREATE_FORK, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return create_fork(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::DELETE_FORK, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return delete_fork(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::FINALIZE_BLOCKS, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return set_finalized(obj, buffer); }); - - _dispatcher.register_target(WorldStateMessageType::UNWIND_BLOCKS, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return unwind(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::REMOVE_HISTORICAL_BLOCKS, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return remove_historical(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::GET_STATUS, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return get_status(obj, buffer); }); - - _dispatcher.register_target(WorldStateMessageType::CLOSE, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return close(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::CREATE_CHECKPOINT, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return checkpoint(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::COMMIT_CHECKPOINT, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return commit_checkpoint(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::REVERT_CHECKPOINT, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return revert_checkpoint(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::COMMIT_ALL_CHECKPOINTS, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return commit_all_checkpoints_to(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::REVERT_ALL_CHECKPOINTS, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return revert_all_checkpoints_to(obj, buffer); }); - - _dispatcher.register_target( - WorldStateMessageType::COPY_STORES, - [this](msgpack::object& obj, msgpack::sbuffer& buffer) { return copy_stores(obj, buffer); }); -} - -Napi::Value WorldStateWrapper::call(const Napi::CallbackInfo& info) -{ - Napi::Env env = info.Env(); - // keep this in a shared pointer so that AsyncOperation can resolve/reject the promise once the execution is - // complete on an separate thread - auto deferred = std::make_shared(env); - - if (info.Length() < 1) { - deferred->Reject(Napi::TypeError::New(env, "Wrong number of arguments").Value()); - } else if (!info[0].IsBuffer()) { - deferred->Reject(Napi::TypeError::New(env, "Argument must be a buffer").Value()); - } else if (!_ws) { - deferred->Reject(Napi::TypeError::New(env, "World state has been closed").Value()); - } else { - auto buffer = info[0].As>(); - size_t length = buffer.Length(); - // we mustn't access the Napi::Env outside of this top-level function - // so copy the data to a variable we own - // and make it a shared pointer so that it doesn't get destroyed as soon as we exit this code block - auto data = std::make_shared>(length); - std::copy_n(buffer.Data(), length, data->data()); - - auto* op = new AsyncOperation(env, deferred, [=, this](msgpack::sbuffer& buf) { - msgpack::object_handle obj_handle = msgpack::unpack(data->data(), length); - msgpack::object obj = obj_handle.get(); - _dispatcher.on_new_data(obj, buf); - }); - - // Napi is now responsible for destroying this object - op->Queue(); - } - - return deferred->Promise(); -} - -Napi::Value WorldStateWrapper::getHandle(const Napi::CallbackInfo& info) -{ - Napi::Env env = info.Env(); - - if (!_ws) { - throw Napi::Error::New(env, "World state has been closed"); - } - - // Return a NAPI External that wraps the raw WorldState pointer - // This allows other NAPI functions to access the WorldState instance - return Napi::External::New(env, _ws.get()); -} - -bool WorldStateWrapper::get_tree_info(msgpack::object& obj, msgpack::sbuffer& buffer) const -{ - TypedMessage request; - obj.convert(request); - auto info = _ws->get_tree_info(request.value.revision, request.value.treeId); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg( - WorldStateMessageType::GET_TREE_INFO, - header, - { request.value.treeId, info.meta.root, info.meta.size, info.meta.depth }); - - msgpack::pack(buffer, resp_msg); - - return true; -} - -bool WorldStateWrapper::get_state_reference(msgpack::object& obj, msgpack::sbuffer& buffer) const -{ - TypedMessage request; - obj.convert(request); - auto state = _ws->get_state_reference(request.value.revision); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg( - WorldStateMessageType::GET_STATE_REFERENCE, header, { state }); - - msgpack::pack(buffer, resp_msg); - - return true; -} - -bool WorldStateWrapper::get_initial_state_reference(msgpack::object& obj, msgpack::sbuffer& buffer) const -{ - HeaderOnlyMessage request; - obj.convert(request); - auto state = _ws->get_initial_state_reference(); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg( - WorldStateMessageType::GET_INITIAL_STATE_REFERENCE, header, { state }); - - msgpack::pack(buffer, resp_msg); - - return true; -} - -bool WorldStateWrapper::get_leaf_value(msgpack::object& obj, msgpack::sbuffer& buffer) const -{ - TypedMessage request; - obj.convert(request); - - switch (request.value.treeId) { - case MerkleTreeId::NOTE_HASH_TREE: - case MerkleTreeId::L1_TO_L2_MESSAGE_TREE: - case MerkleTreeId::ARCHIVE: { - auto leaf = _ws->get_leaf(request.value.revision, request.value.treeId, request.value.leafIndex); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage> resp_msg(WorldStateMessageType::GET_LEAF_VALUE, header, leaf); - msgpack::pack(buffer, resp_msg); - break; - } - - case MerkleTreeId::PUBLIC_DATA_TREE: { - auto leaf = _ws->get_leaf( - request.value.revision, request.value.treeId, request.value.leafIndex); - MsgHeader header(request.header.messageId); - messaging::TypedMessage> resp_msg( - WorldStateMessageType::GET_LEAF_VALUE, header, leaf); - msgpack::pack(buffer, resp_msg); - break; - } - - case MerkleTreeId::NULLIFIER_TREE: { - auto leaf = _ws->get_leaf( - request.value.revision, request.value.treeId, request.value.leafIndex); - MsgHeader header(request.header.messageId); - messaging::TypedMessage> resp_msg( - WorldStateMessageType::GET_LEAF_VALUE, header, leaf); - msgpack::pack(buffer, resp_msg); - break; - } - - default: - throw std::runtime_error("Unsupported tree type"); - } - - return true; -} - -bool WorldStateWrapper::get_leaf_preimage(msgpack::object& obj, msgpack::sbuffer& buffer) const -{ - TypedMessage request; - obj.convert(request); - - MsgHeader header(request.header.messageId); - - switch (request.value.treeId) { - case MerkleTreeId::NULLIFIER_TREE: { - auto leaf = _ws->get_indexed_leaf( - request.value.revision, request.value.treeId, request.value.leafIndex); - messaging::TypedMessage>> resp_msg( - WorldStateMessageType::GET_LEAF_PREIMAGE, header, leaf); - msgpack::pack(buffer, resp_msg); - break; - } - - case MerkleTreeId::PUBLIC_DATA_TREE: { - auto leaf = _ws->get_indexed_leaf( - request.value.revision, request.value.treeId, request.value.leafIndex); - - messaging::TypedMessage>> resp_msg( - WorldStateMessageType::GET_LEAF_PREIMAGE, header, leaf); - msgpack::pack(buffer, resp_msg); - break; - } - - default: - throw std::runtime_error("Unsupported tree type"); - } - - return true; -} - -bool WorldStateWrapper::get_sibling_path(msgpack::object& obj, msgpack::sbuffer& buffer) const -{ - TypedMessage request; - obj.convert(request); - - fr_sibling_path path = _ws->get_sibling_path(request.value.revision, request.value.treeId, request.value.leafIndex); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg(WorldStateMessageType::GET_SIBLING_PATH, header, path); - - msgpack::pack(buffer, resp_msg); - - return true; -} - -bool WorldStateWrapper::get_block_numbers_for_leaf_indices(msgpack::object& obj, msgpack::sbuffer& buffer) const -{ - TypedMessage request; - obj.convert(request); - - GetBlockNumbersForLeafIndicesResponse response; - _ws->get_block_numbers_for_leaf_indices( - request.value.revision, request.value.treeId, request.value.leafIndices, response.blockNumbers); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg( - WorldStateMessageType::GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, header, response); - - msgpack::pack(buffer, resp_msg); - - return true; -} - -bool WorldStateWrapper::find_leaf_indices(msgpack::object& obj, msgpack::sbuffer& buffer) const -{ - TypedMessage request; - obj.convert(request); - - FindLeafIndicesResponse response; - - switch (request.value.treeId) { - case MerkleTreeId::NOTE_HASH_TREE: - case MerkleTreeId::L1_TO_L2_MESSAGE_TREE: - case MerkleTreeId::ARCHIVE: { - TypedMessage> r1; - obj.convert(r1); - _ws->find_leaf_indices( - request.value.revision, request.value.treeId, r1.value.leaves, response.indices, r1.value.startIndex); - break; - } - - case MerkleTreeId::PUBLIC_DATA_TREE: { - TypedMessage> r2; - obj.convert(r2); - _ws->find_leaf_indices( - request.value.revision, request.value.treeId, r2.value.leaves, response.indices, r2.value.startIndex); - break; - } - case MerkleTreeId::NULLIFIER_TREE: { - TypedMessage> r3; - obj.convert(r3); - _ws->find_leaf_indices( - request.value.revision, request.value.treeId, r3.value.leaves, response.indices, r3.value.startIndex); - break; - } - default: - throw std::runtime_error("Unsupported tree type"); - } - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg( - WorldStateMessageType::FIND_LEAF_INDICES, header, response); - msgpack::pack(buffer, resp_msg); - - return true; -} - -bool WorldStateWrapper::find_sibling_paths(msgpack::object& obj, msgpack::sbuffer& buffer) const -{ - TypedMessage request; - obj.convert(request); - - FindLeafPathsResponse response; - - switch (request.value.treeId) { - case MerkleTreeId::NOTE_HASH_TREE: - case MerkleTreeId::L1_TO_L2_MESSAGE_TREE: - case MerkleTreeId::ARCHIVE: { - TypedMessage> r1; - obj.convert(r1); - _ws->find_sibling_paths(request.value.revision, request.value.treeId, r1.value.leaves, response.paths); - break; - } - - case MerkleTreeId::PUBLIC_DATA_TREE: { - TypedMessage> r2; - obj.convert(r2); - _ws->find_sibling_paths( - request.value.revision, request.value.treeId, r2.value.leaves, response.paths); - break; - } - case MerkleTreeId::NULLIFIER_TREE: { - TypedMessage> r3; - obj.convert(r3); - _ws->find_sibling_paths( - request.value.revision, request.value.treeId, r3.value.leaves, response.paths); - break; - } - default: - throw std::runtime_error("Unsupported tree type"); - } - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg( - WorldStateMessageType::FIND_SIBLING_PATHS, header, response); - msgpack::pack(buffer, resp_msg); - - return true; -} - -bool WorldStateWrapper::find_low_leaf(msgpack::object& obj, msgpack::sbuffer& buffer) const -{ - TypedMessage request; - obj.convert(request); - - GetLowIndexedLeafResponse low_leaf_info = - _ws->find_low_leaf_index(request.value.revision, request.value.treeId, request.value.key); - - MsgHeader header(request.header.messageId); - TypedMessage response( - WorldStateMessageType::FIND_LOW_LEAF, header, { low_leaf_info.is_already_present, low_leaf_info.index }); - msgpack::pack(buffer, response); - - return true; -} - -bool WorldStateWrapper::append_leaves(msgpack::object& obj, msgpack::sbuffer& buf) -{ - TypedMessage request; - obj.convert(request); - - switch (request.value.treeId) { - case MerkleTreeId::NOTE_HASH_TREE: - case MerkleTreeId::L1_TO_L2_MESSAGE_TREE: - case MerkleTreeId::ARCHIVE: { - TypedMessage> r1; - obj.convert(r1); - _ws->append_leaves(r1.value.treeId, r1.value.leaves, r1.value.forkId); - break; - } - case MerkleTreeId::PUBLIC_DATA_TREE: { - TypedMessage> r2; - obj.convert(r2); - _ws->append_leaves(r2.value.treeId, r2.value.leaves, r2.value.forkId); - break; - } - case MerkleTreeId::NULLIFIER_TREE: { - TypedMessage> r3; - obj.convert(r3); - _ws->append_leaves(r3.value.treeId, r3.value.leaves, r3.value.forkId); - break; - } - default: - throw std::runtime_error("Unsupported tree type"); - } - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg(WorldStateMessageType::APPEND_LEAVES, header, {}); - msgpack::pack(buf, resp_msg); - - return true; -} - -bool WorldStateWrapper::batch_insert(msgpack::object& obj, msgpack::sbuffer& buffer) -{ - TypedMessage request; - obj.convert(request); - - switch (request.value.treeId) { - case MerkleTreeId::PUBLIC_DATA_TREE: { - TypedMessage> r1; - obj.convert(r1); - auto result = _ws->batch_insert_indexed_leaves( - request.value.treeId, r1.value.leaves, r1.value.subtreeDepth, r1.value.forkId); - MsgHeader header(request.header.messageId); - messaging::TypedMessage> resp_msg( - WorldStateMessageType::BATCH_INSERT, header, result); - msgpack::pack(buffer, resp_msg); - - break; - } - case MerkleTreeId::NULLIFIER_TREE: { - TypedMessage> r2; - obj.convert(r2); - auto result = _ws->batch_insert_indexed_leaves( - request.value.treeId, r2.value.leaves, r2.value.subtreeDepth, r2.value.forkId); - MsgHeader header(request.header.messageId); - messaging::TypedMessage> resp_msg( - WorldStateMessageType::BATCH_INSERT, header, result); - msgpack::pack(buffer, resp_msg); - break; - } - default: - throw std::runtime_error("Unsupported tree type"); - } - - return true; -} - -bool WorldStateWrapper::sequential_insert(msgpack::object& obj, msgpack::sbuffer& buffer) -{ - TypedMessage request; - obj.convert(request); - - switch (request.value.treeId) { - case MerkleTreeId::PUBLIC_DATA_TREE: { - TypedMessage> r1; - obj.convert(r1); - auto result = _ws->insert_indexed_leaves( - request.value.treeId, r1.value.leaves, r1.value.forkId); - MsgHeader header(request.header.messageId); - messaging::TypedMessage> resp_msg( - WorldStateMessageType::SEQUENTIAL_INSERT, header, result); - msgpack::pack(buffer, resp_msg); - - break; - } - case MerkleTreeId::NULLIFIER_TREE: { - TypedMessage> r2; - obj.convert(r2); - auto result = _ws->insert_indexed_leaves( - request.value.treeId, r2.value.leaves, r2.value.forkId); - MsgHeader header(request.header.messageId); - messaging::TypedMessage> resp_msg( - WorldStateMessageType::SEQUENTIAL_INSERT, header, result); - msgpack::pack(buffer, resp_msg); - break; - } - default: - throw std::runtime_error("Unsupported tree type"); - } - - return true; -} - -bool WorldStateWrapper::update_archive(msgpack::object& obj, msgpack::sbuffer& buf) -{ - TypedMessage request; - obj.convert(request); - - _ws->update_archive(request.value.blockStateRef, request.value.blockHeaderHash, request.value.forkId); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg(WorldStateMessageType::UPDATE_ARCHIVE, header, {}); - msgpack::pack(buf, resp_msg); - - return true; -} - -bool WorldStateWrapper::commit(msgpack::object& obj, msgpack::sbuffer& buf) -{ - HeaderOnlyMessage request; - obj.convert(request); - - WorldStateStatusFull status; - _ws->commit(status); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg(WorldStateMessageType::COMMIT, header, { status }); - msgpack::pack(buf, resp_msg); - - return true; -} - -bool WorldStateWrapper::rollback(msgpack::object& obj, msgpack::sbuffer& buf) -{ - HeaderOnlyMessage request; - obj.convert(request); - - _ws->rollback(); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg(WorldStateMessageType::ROLLBACK, header, {}); - msgpack::pack(buf, resp_msg); - - return true; -} - -bool WorldStateWrapper::sync_block(msgpack::object& obj, msgpack::sbuffer& buf) -{ - TypedMessage request; - obj.convert(request); - - WorldStateStatusFull status = _ws->sync_block(request.value.blockStateRef, - request.value.blockHeaderHash, - request.value.paddedNoteHashes, - request.value.paddedL1ToL2Messages, - request.value.paddedNullifiers, - request.value.publicDataWrites); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg(WorldStateMessageType::SYNC_BLOCK, header, { status }); - msgpack::pack(buf, resp_msg); - - return true; -} - -bool WorldStateWrapper::create_fork(msgpack::object& obj, msgpack::sbuffer& buf) -{ - TypedMessage request; - obj.convert(request); - - std::optional blockNumber = - request.value.latest ? std::nullopt : std::optional(request.value.blockNumber); - - uint64_t forkId = _ws->create_fork(blockNumber); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg(WorldStateMessageType::CREATE_FORK, header, { forkId }); - msgpack::pack(buf, resp_msg); - - return true; -} - -bool WorldStateWrapper::delete_fork(msgpack::object& obj, msgpack::sbuffer& buf) -{ - TypedMessage request; - obj.convert(request); - - _ws->delete_fork(request.value.forkId); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg(WorldStateMessageType::DELETE_FORK, header, {}); - msgpack::pack(buf, resp_msg); - - return true; -} - -bool WorldStateWrapper::close(msgpack::object& obj, msgpack::sbuffer& buf) -{ - HeaderOnlyMessage request; - obj.convert(request); - - // The only reason this API exists is for testing purposes in TS (e.g. close db, open new db instance to test - // persistence) - _ws.reset(nullptr); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg(WorldStateMessageType::CLOSE, header, {}); - msgpack::pack(buf, resp_msg); - - return true; -} - -bool WorldStateWrapper::set_finalized(msgpack::object& obj, msgpack::sbuffer& buf) const -{ - TypedMessage request; - obj.convert(request); - WorldStateStatusSummary status = _ws->set_finalized_blocks(request.value.toBlockNumber); - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg( - WorldStateMessageType::FINALIZE_BLOCKS, header, { status }); - msgpack::pack(buf, resp_msg); - - return true; -} - -bool WorldStateWrapper::unwind(msgpack::object& obj, msgpack::sbuffer& buf) const -{ - TypedMessage request; - obj.convert(request); - - WorldStateStatusFull status = _ws->unwind_blocks(request.value.toBlockNumber); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg(WorldStateMessageType::UNWIND_BLOCKS, header, { status }); - msgpack::pack(buf, resp_msg); - - return true; -} - -bool WorldStateWrapper::remove_historical(msgpack::object& obj, msgpack::sbuffer& buf) const -{ - TypedMessage request; - obj.convert(request); - WorldStateStatusFull status = _ws->remove_historical_blocks(request.value.toBlockNumber); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg( - WorldStateMessageType::REMOVE_HISTORICAL_BLOCKS, header, { status }); - msgpack::pack(buf, resp_msg); - - return true; -} - -bool WorldStateWrapper::checkpoint(msgpack::object& obj, msgpack::sbuffer& buffer) -{ - TypedMessage request; - obj.convert(request); - - uint32_t depth = _ws->checkpoint(request.value.forkId); - - MsgHeader header(request.header.messageId); - CheckpointDepthResponse resp_value{ depth }; - messaging::TypedMessage resp_msg( - WorldStateMessageType::CREATE_CHECKPOINT, header, resp_value); - msgpack::pack(buffer, resp_msg); - - return true; -} - -bool WorldStateWrapper::commit_checkpoint(msgpack::object& obj, msgpack::sbuffer& buffer) -{ - TypedMessage request; - obj.convert(request); - - _ws->commit_checkpoint(request.value.forkId); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg(WorldStateMessageType::COMMIT_CHECKPOINT, header, {}); - msgpack::pack(buffer, resp_msg); - - return true; -} - -bool WorldStateWrapper::revert_checkpoint(msgpack::object& obj, msgpack::sbuffer& buffer) -{ - TypedMessage request; - obj.convert(request); - - _ws->revert_checkpoint(request.value.forkId); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg(WorldStateMessageType::REVERT_CHECKPOINT, header, {}); - msgpack::pack(buffer, resp_msg); - - return true; -} - -bool WorldStateWrapper::commit_all_checkpoints_to(msgpack::object& obj, msgpack::sbuffer& buffer) -{ - TypedMessage request; - obj.convert(request); - - _ws->commit_all_checkpoints_to(request.value.forkId, request.value.depth); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg(WorldStateMessageType::COMMIT_ALL_CHECKPOINTS, header, {}); - msgpack::pack(buffer, resp_msg); - - return true; -} - -bool WorldStateWrapper::revert_all_checkpoints_to(msgpack::object& obj, msgpack::sbuffer& buffer) -{ - TypedMessage request; - obj.convert(request); - - _ws->revert_all_checkpoints_to(request.value.forkId, request.value.depth); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg(WorldStateMessageType::REVERT_ALL_CHECKPOINTS, header, {}); - msgpack::pack(buffer, resp_msg); - - return true; -} - -bool WorldStateWrapper::get_status(msgpack::object& obj, msgpack::sbuffer& buf) const -{ - HeaderOnlyMessage request; - obj.convert(request); - - WorldStateStatusSummary status; - _ws->get_status_summary(status); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg(WorldStateMessageType::GET_STATUS, header, { status }); - msgpack::pack(buf, resp_msg); - - return true; -} - -bool WorldStateWrapper::copy_stores(msgpack::object& obj, msgpack::sbuffer& buffer) -{ - TypedMessage request; - obj.convert(request); - - _ws->copy_stores(request.value.dstPath, request.value.compact.value_or(false)); - - MsgHeader header(request.header.messageId); - messaging::TypedMessage resp_msg(WorldStateMessageType::COPY_STORES, header, {}); - msgpack::pack(buffer, resp_msg); - - return true; -} - -Napi::Function WorldStateWrapper::get_class(Napi::Env env) -{ - return DefineClass(env, - "WorldState", - { - WorldStateWrapper::InstanceMethod("call", &WorldStateWrapper::call), - WorldStateWrapper::InstanceMethod("getHandle", &WorldStateWrapper::getHandle), - }); -} diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.hpp b/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.hpp deleted file mode 100644 index cd4f0d02e8e1..000000000000 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state.hpp +++ /dev/null @@ -1,84 +0,0 @@ -#pragma once - -#include "barretenberg/messaging/dispatcher.hpp" -#include "barretenberg/nodejs_module/world_state/world_state_message.hpp" -#include "barretenberg/world_state/types.hpp" -#include "barretenberg/world_state/world_state.hpp" -#include -#include -#include - -namespace bb::nodejs { - -/** - * @brief Manages the interaction between the JavaScript runtime and the WorldState class. - */ -class WorldStateWrapper : public Napi::ObjectWrap { - public: - WorldStateWrapper(const Napi::CallbackInfo&); - - /** - * @brief The only instance method exposed to JavaScript. Takes a msgpack Message and returns a Promise - */ - Napi::Value call(const Napi::CallbackInfo&); - - /** - * @brief Get a NAPI External handle to the underlying WorldState pointer. - * This allows other NAPI functions to access the WorldState instance directly. - */ - Napi::Value getHandle(const Napi::CallbackInfo&); - - /** - * @brief Register the WorldStateAddon class with the JavaScript runtime. - */ - static Napi::Function get_class(Napi::Env); - - private: - std::unique_ptr _ws; - bb::messaging::MessageDispatcher _dispatcher; - - bool get_tree_info(msgpack::object& obj, msgpack::sbuffer& buffer) const; - bool get_state_reference(msgpack::object& obj, msgpack::sbuffer& buffer) const; - bool get_initial_state_reference(msgpack::object& obj, msgpack::sbuffer& buffer) const; - - bool get_leaf_value(msgpack::object& obj, msgpack::sbuffer& buffer) const; - bool get_leaf_preimage(msgpack::object& obj, msgpack::sbuffer& buffer) const; - bool get_sibling_path(msgpack::object& obj, msgpack::sbuffer& buffer) const; - bool get_block_numbers_for_leaf_indices(msgpack::object& obj, msgpack::sbuffer& buffer) const; - - bool find_leaf_indices(msgpack::object& obj, msgpack::sbuffer& buffer) const; - bool find_low_leaf(msgpack::object& obj, msgpack::sbuffer& buffer) const; - bool find_sibling_paths(msgpack::object& obj, msgpack::sbuffer& buffer) const; - - bool append_leaves(msgpack::object& obj, msgpack::sbuffer& buffer); - bool batch_insert(msgpack::object& obj, msgpack::sbuffer& buffer); - bool sequential_insert(msgpack::object& obj, msgpack::sbuffer& buffer); - - bool update_archive(msgpack::object& obj, msgpack::sbuffer& buffer); - - bool commit(msgpack::object& obj, msgpack::sbuffer& buffer); - bool rollback(msgpack::object& obj, msgpack::sbuffer& buffer); - - bool sync_block(msgpack::object& obj, msgpack::sbuffer& buffer); - - bool create_fork(msgpack::object& obj, msgpack::sbuffer& buffer); - bool delete_fork(msgpack::object& obj, msgpack::sbuffer& buffer); - - bool close(msgpack::object& obj, msgpack::sbuffer& buffer); - - bool set_finalized(msgpack::object& obj, msgpack::sbuffer& buffer) const; - bool unwind(msgpack::object& obj, msgpack::sbuffer& buffer) const; - bool remove_historical(msgpack::object& obj, msgpack::sbuffer& buffer) const; - - bool get_status(msgpack::object& obj, msgpack::sbuffer& buffer) const; - - bool checkpoint(msgpack::object& obj, msgpack::sbuffer& buffer); - bool commit_checkpoint(msgpack::object& obj, msgpack::sbuffer& buffer); - bool revert_checkpoint(msgpack::object& obj, msgpack::sbuffer& buffer); - bool commit_all_checkpoints_to(msgpack::object& obj, msgpack::sbuffer& buffer); - bool revert_all_checkpoints_to(msgpack::object& obj, msgpack::sbuffer& buffer); - - bool copy_stores(msgpack::object& obj, msgpack::sbuffer& buffer); -}; - -} // namespace bb::nodejs diff --git a/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state_message.hpp b/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state_message.hpp deleted file mode 100644 index 8f6b481ad41a..000000000000 --- a/barretenberg/cpp/src/barretenberg/nodejs_module/world_state/world_state_message.hpp +++ /dev/null @@ -1,272 +0,0 @@ -#pragma once -#include "barretenberg/crypto/merkle_tree/hash_path.hpp" -#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp" -#include "barretenberg/crypto/merkle_tree/response.hpp" -#include "barretenberg/crypto/merkle_tree/types.hpp" -#include "barretenberg/ecc/curves/bn254/fr.hpp" -#include "barretenberg/messaging/header.hpp" -#include "barretenberg/serialize/msgpack.hpp" -#include "barretenberg/world_state/fork.hpp" -#include "barretenberg/world_state/types.hpp" -#include -#include -#include - -namespace bb::nodejs { - -using namespace bb::messaging; -using namespace bb::world_state; - -enum WorldStateMessageType { - GET_TREE_INFO = FIRST_APP_MSG_TYPE, - GET_STATE_REFERENCE, - GET_INITIAL_STATE_REFERENCE, - - GET_LEAF_VALUE, - GET_LEAF_PREIMAGE, - GET_SIBLING_PATH, - GET_BLOCK_NUMBERS_FOR_LEAF_INDICES, - - FIND_LEAF_INDICES, - FIND_LOW_LEAF, - FIND_SIBLING_PATHS, - - APPEND_LEAVES, - BATCH_INSERT, - SEQUENTIAL_INSERT, - - UPDATE_ARCHIVE, - - COMMIT, - ROLLBACK, - - SYNC_BLOCK, - - CREATE_FORK, - DELETE_FORK, - - FINALIZE_BLOCKS, - UNWIND_BLOCKS, - REMOVE_HISTORICAL_BLOCKS, - - GET_STATUS, - - CREATE_CHECKPOINT, - COMMIT_CHECKPOINT, - REVERT_CHECKPOINT, - COMMIT_ALL_CHECKPOINTS, - REVERT_ALL_CHECKPOINTS, - - COPY_STORES, - - CLOSE = 999, -}; - -struct TreeIdOnlyRequest { - MerkleTreeId treeId; - SERIALIZATION_FIELDS(treeId); -}; - -struct CreateForkRequest { - bool latest; - block_number_t blockNumber; - SERIALIZATION_FIELDS(latest, blockNumber); -}; - -struct CreateForkResponse { - uint64_t forkId; - SERIALIZATION_FIELDS(forkId); -}; - -struct DeleteForkRequest { - uint64_t forkId; - SERIALIZATION_FIELDS(forkId); -}; - -struct ForkIdOnlyRequest { - uint64_t forkId; - SERIALIZATION_FIELDS(forkId); -}; - -struct ForkIdWithDepthRequest { - uint64_t forkId; - uint32_t depth; - SERIALIZATION_FIELDS(forkId, depth); -}; - -struct CheckpointDepthResponse { - uint32_t depth; - SERIALIZATION_FIELDS(depth); -}; - -struct TreeIdAndRevisionRequest { - MerkleTreeId treeId; - WorldStateRevision revision; - SERIALIZATION_FIELDS(treeId, revision); -}; - -struct EmptyResponse { - bool ok{ true }; - SERIALIZATION_FIELDS(ok); -}; - -struct GetTreeInfoRequest { - MerkleTreeId treeId; - WorldStateRevision revision; - SERIALIZATION_FIELDS(treeId, revision); -}; - -struct GetTreeInfoResponse { - MerkleTreeId treeId; - fr root; - index_t size; - uint32_t depth; - SERIALIZATION_FIELDS(treeId, root, size, depth); -}; - -struct GetStateReferenceRequest { - WorldStateRevision revision; - SERIALIZATION_FIELDS(revision); -}; - -struct GetStateReferenceResponse { - StateReference state; - SERIALIZATION_FIELDS(state); -}; - -struct GetInitialStateReferenceResponse { - StateReference state; - SERIALIZATION_FIELDS(state); -}; - -struct GetLeafValueRequest { - MerkleTreeId treeId; - WorldStateRevision revision; - index_t leafIndex; - SERIALIZATION_FIELDS(treeId, revision, leafIndex); -}; - -struct GetLeafPreimageRequest { - MerkleTreeId treeId; - WorldStateRevision revision; - index_t leafIndex; - SERIALIZATION_FIELDS(treeId, revision, leafIndex); -}; - -struct GetSiblingPathRequest { - MerkleTreeId treeId; - WorldStateRevision revision; - index_t leafIndex; - SERIALIZATION_FIELDS(treeId, revision, leafIndex); -}; - -struct GetBlockNumbersForLeafIndicesRequest { - MerkleTreeId treeId; - WorldStateRevision revision; - std::vector leafIndices; - SERIALIZATION_FIELDS(treeId, revision, leafIndices); -}; - -struct GetBlockNumbersForLeafIndicesResponse { - std::vector> blockNumbers; - SERIALIZATION_FIELDS(blockNumbers); -}; - -template struct FindLeafIndicesRequest { - MerkleTreeId treeId; - WorldStateRevision revision; - std::vector leaves; - index_t startIndex; - SERIALIZATION_FIELDS(treeId, revision, leaves, startIndex); -}; - -struct FindLeafIndicesResponse { - std::vector> indices; - SERIALIZATION_FIELDS(indices); -}; - -template struct FindLeafPathsRequest { - MerkleTreeId treeId; - WorldStateRevision revision; - std::vector leaves; - SERIALIZATION_FIELDS(treeId, revision, leaves); -}; - -struct FindLeafPathsResponse { - std::vector> paths; - SERIALIZATION_FIELDS(paths); -}; - -struct FindLowLeafRequest { - MerkleTreeId treeId; - WorldStateRevision revision; - fr key; - SERIALIZATION_FIELDS(treeId, revision, key); -}; - -struct FindLowLeafResponse { - bool alreadyPresent; - index_t index; - SERIALIZATION_FIELDS(alreadyPresent, index); -}; - -struct BlockShiftRequest { - block_number_t toBlockNumber; - SERIALIZATION_FIELDS(toBlockNumber); -}; - -template struct AppendLeavesRequest { - MerkleTreeId treeId; - std::vector leaves; - Fork::Id forkId{ CANONICAL_FORK_ID }; - SERIALIZATION_FIELDS(treeId, leaves, forkId); -}; - -template struct BatchInsertRequest { - MerkleTreeId treeId; - std::vector leaves; - uint32_t subtreeDepth; - Fork::Id forkId{ CANONICAL_FORK_ID }; - SERIALIZATION_FIELDS(treeId, leaves, subtreeDepth, forkId); -}; - -template struct InsertRequest { - MerkleTreeId treeId; - std::vector leaves; - Fork::Id forkId{ CANONICAL_FORK_ID }; - SERIALIZATION_FIELDS(treeId, leaves, forkId); -}; - -struct UpdateArchiveRequest { - StateReference blockStateRef; - bb::fr blockHeaderHash; - Fork::Id forkId{ CANONICAL_FORK_ID }; - SERIALIZATION_FIELDS(blockStateRef, blockHeaderHash, forkId); -}; - -struct SyncBlockRequest { - block_number_t blockNumber; - StateReference blockStateRef; - bb::fr blockHeaderHash; - std::vector paddedNoteHashes, paddedL1ToL2Messages; - std::vector paddedNullifiers; - std::vector publicDataWrites; - - SERIALIZATION_FIELDS(blockNumber, - blockStateRef, - blockHeaderHash, - paddedNoteHashes, - paddedL1ToL2Messages, - paddedNullifiers, - publicDataWrites); -}; - -struct CopyStoresRequest { - std::string dstPath; - std::optional compact; - SERIALIZATION_FIELDS(dstPath, compact); -}; - -} // namespace bb::nodejs - -MSGPACK_ADD_ENUM(bb::nodejs::WorldStateMessageType) diff --git a/barretenberg/cpp/src/barretenberg/vm2/avm_sim_api.cpp b/barretenberg/cpp/src/barretenberg/vm2/avm_sim_api.cpp index 2d706a15e64a..d16dc16bc698 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/avm_sim_api.cpp +++ b/barretenberg/cpp/src/barretenberg/vm2/avm_sim_api.cpp @@ -10,7 +10,7 @@ using namespace bb::avm2::simulation; TxSimulationResult AvmSimAPI::simulate(const FastSimulationInputs& inputs, simulation::ContractDBInterface& contract_db, - world_state::WorldState& ws, + simulation::LowLevelMerkleDBInterface& merkle_db, simulation::CancellationTokenPtr cancellation_token) { vinfo("Simulating..."); @@ -18,25 +18,23 @@ TxSimulationResult AvmSimAPI::simulate(const FastSimulationInputs& inputs, if (inputs.config.collect_hints) { return AVM_TRACK_TIME_V("simulation/all", - simulation_helper.simulate_for_hint_collection(contract_db, - inputs.ws_revision, - ws, - inputs.config, - inputs.tx, - inputs.global_variables, - inputs.protocol_contracts, - cancellation_token)); - } else { - return AVM_TRACK_TIME_V("simulation/all", - simulation_helper.simulate_fast_with_existing_ws(contract_db, - inputs.ws_revision, - ws, - inputs.config, - inputs.tx, - inputs.global_variables, - inputs.protocol_contracts, - cancellation_token)); + simulation_helper.simulate_for_hint_collection_internal(contract_db, + merkle_db, + inputs.config, + inputs.tx, + inputs.global_variables, + inputs.protocol_contracts, + cancellation_token)); } + + return AVM_TRACK_TIME_V("simulation/all", + simulation_helper.simulate_fast_internal(contract_db, + merkle_db, + inputs.config, + inputs.tx, + inputs.global_variables, + inputs.protocol_contracts, + cancellation_token)); } TxSimulationResult AvmSimAPI::simulate_with_hinted_dbs(const ProvingInputs& inputs) diff --git a/barretenberg/cpp/src/barretenberg/vm2/avm_sim_api.hpp b/barretenberg/cpp/src/barretenberg/vm2/avm_sim_api.hpp index 77fb9c977a74..c1b496327d1c 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/avm_sim_api.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2/avm_sim_api.hpp @@ -15,7 +15,7 @@ class AvmSimAPI { TxSimulationResult simulate(const FastSimulationInputs& inputs, simulation::ContractDBInterface& contract_db, - world_state::WorldState& ws, + simulation::LowLevelMerkleDBInterface& merkle_db, simulation::CancellationTokenPtr cancellation_token = nullptr); TxSimulationResult simulate_with_hinted_dbs(const AvmProvingInputs& inputs); }; diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation_helper.cpp b/barretenberg/cpp/src/barretenberg/vm2/simulation_helper.cpp index 0bc66fc810a0..ced8480f4a01 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation_helper.cpp +++ b/barretenberg/cpp/src/barretenberg/vm2/simulation_helper.cpp @@ -576,21 +576,17 @@ TxSimulationResult AvmSimulationHelper::simulate_fast_with_existing_ws( raw_contract_db, raw_merkle_db, config, tx, global_variables, protocol_contracts, cancellation_token); } -TxSimulationResult AvmSimulationHelper::simulate_for_hint_collection( +TxSimulationResult AvmSimulationHelper::simulate_for_hint_collection_internal( simulation::ContractDBInterface& raw_contract_db, - const world_state::WorldStateRevision& world_state_revision, - world_state::WorldState& ws, + simulation::LowLevelMerkleDBInterface& raw_merkle_db, const PublicSimulatorConfig& config, const Tx& tx, const GlobalVariables& global_variables, const ProtocolContracts& protocol_contracts, CancellationTokenPtr cancellation_token) { - // If you are not collecting hints, don't use this method. - BB_ASSERT(config.collect_hints && "Use simulate_fast_with_existing_ws instead"); - - // Create PureRawMerkleDB with the provided WorldState instance and cancellation token - PureRawMerkleDB raw_merkle_db(world_state_revision, ws, /*cache_tree_roots=*/true, cancellation_token); + (void)cancellation_token; // Not yet used in this path. + BB_ASSERT(config.collect_hints && "Use simulate_fast_internal instead"); auto starting_tree_roots = raw_merkle_db.get_tree_roots(); HintingContractsDB hinting_contract_db(raw_contract_db); @@ -609,11 +605,29 @@ TxSimulationResult AvmSimulationHelper::simulate_for_hint_collection( tx_result.hints = std::move(collected_hints); - // Need to std::move to avoid copying (due to structured bindings). - // This was fixed in C++23 via http://wg21.link/P2266R3. return std::move(tx_result); } +TxSimulationResult AvmSimulationHelper::simulate_for_hint_collection( + simulation::ContractDBInterface& raw_contract_db, + const world_state::WorldStateRevision& world_state_revision, + world_state::WorldState& ws, + const PublicSimulatorConfig& config, + const Tx& tx, + const GlobalVariables& global_variables, + const ProtocolContracts& protocol_contracts, + CancellationTokenPtr cancellation_token) +{ + // If you are not collecting hints, don't use this method. + BB_ASSERT(config.collect_hints && "Use simulate_fast_with_existing_ws instead"); + + // Create PureRawMerkleDB with the provided WorldState instance and cancellation token + PureRawMerkleDB raw_merkle_db(world_state_revision, ws, /*cache_tree_roots=*/true, cancellation_token); + + return simulate_for_hint_collection_internal( + raw_contract_db, raw_merkle_db, config, tx, global_variables, protocol_contracts, cancellation_token); +} + EventsContainer AvmSimulationHelper::simulate_for_witgen(const ExecutionHints& hints) { // TODO(fcarreiro): decide if we want to pass a config here. diff --git a/barretenberg/cpp/src/barretenberg/vm2/simulation_helper.hpp b/barretenberg/cpp/src/barretenberg/vm2/simulation_helper.hpp index 6bae3274dec9..dcb665e814dd 100644 --- a/barretenberg/cpp/src/barretenberg/vm2/simulation_helper.hpp +++ b/barretenberg/cpp/src/barretenberg/vm2/simulation_helper.hpp @@ -37,7 +37,9 @@ class AvmSimulationHelper { // An extra entry point that is not used in production. TxSimulationResult simulate_fast_with_hinted_dbs(const ExecutionHints& hints, const PublicSimulatorConfig& config); - protected: + // Fast simulation against any LowLevelMerkleDBInterface implementation (in-process, IPC, or hinted). + // Used by the standalone aztec-avm and the NAPI AVM after the WSDB cutover, both of which + // construct a WSDB-IPC-backed merkle DB rather than using an in-process WorldState reference. TxSimulationResult simulate_fast_internal(simulation::ContractDBInterface& raw_contract_db, simulation::LowLevelMerkleDBInterface& raw_merkle_db, const PublicSimulatorConfig& config, @@ -46,6 +48,20 @@ class AvmSimulationHelper { const ProtocolContracts& protocol_contracts, simulation::CancellationTokenPtr cancellation_token = nullptr); + // Hint-collecting simulation against any LowLevelMerkleDBInterface implementation. Mirrors + // simulate_fast_internal but wraps the DBs in the hinting proxies used by witgen and dumps + // the recorded hints into the result. Used by the prover-node path on both the NAPI AVM + // (after the WSDB cutover) and the standalone aztec-avm. + TxSimulationResult simulate_for_hint_collection_internal( + simulation::ContractDBInterface& raw_contract_db, + simulation::LowLevelMerkleDBInterface& raw_merkle_db, + const PublicSimulatorConfig& config, + const Tx& tx, + const GlobalVariables& global_variables, + const ProtocolContracts& protocol_contracts, + simulation::CancellationTokenPtr cancellation_token = nullptr); + + protected: template