diff --git a/CMakeLists.txt b/CMakeLists.txt index a56a4db40547..ba7dd35f1989 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -121,13 +121,7 @@ if(WITH_USDT) find_package(USDT MODULE REQUIRED) endif() -option(WITH_EXTERNAL_LIBMULTIPROCESS "Build with external libmultiprocess library instead of with local git subtree. This is not normally recommended, but can be useful for developing libmultiprocess itself." OFF) -if(WITH_EXTERNAL_LIBMULTIPROCESS) - find_package(Libmultiprocess REQUIRED COMPONENTS Lib) - find_package(LibmultiprocessNative REQUIRED COMPONENTS Bin - NAMES Libmultiprocess - ) -endif() +find_package(CapnProto REQUIRED) option(BUILD_BENCH "Build bench_bitcoin executable." OFF) option(BUILD_FUZZ_BINARY "Build fuzz binary." OFF) diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index 837e661a9cc9..3774674a049e 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -196,6 +196,14 @@ if [ "${RUN_TIDY}" = "true" ]; then fi if [[ "${RUN_IWYU}" == true ]]; then + mkdir -p "${BASE_BUILD_DIR}/src/node/data" + cmake -DRAW_SOURCE_PATH="${BASE_ROOT_DIR}/src/node/data/ip_asn.dat" \ + -DHEADER_PATH="${BASE_BUILD_DIR}/src/node/data/ip_asn.dat.h" \ + -DRAW_NAMESPACE=node::data \ + -P "${BASE_ROOT_DIR}/cmake/script/GenerateHeaderFromRaw.cmake" + # shellcheck disable=SC2086 + cmake --build "${BASE_BUILD_DIR}" "$MAKEJOBS" --target bitcoin_ipc + # TODO: Consider enforcing IWYU across the entire codebase. FILES_WITH_ENFORCED_IWYU="/src/(((crypto|index|kernel|primitives|script|univalue/(lib|test)|util)/.*|bench/(block_assemble|connectblock)|common/license_info|node/(blockstorage|interfaces|miner|mining_args|utxo_snapshot)|rpc/mining|clientversion|core_io|signet|init)\\.cpp)" jq --arg patterns "$FILES_WITH_ENFORCED_IWYU" 'map(select(.file | test($patterns)))' "${BASE_BUILD_DIR}/compile_commands.json" > "${BASE_BUILD_DIR}/compile_commands_iwyu_errors.json" diff --git a/cmake/libmultiprocess.cmake b/cmake/libmultiprocess.cmake deleted file mode 100644 index 027109a90511..000000000000 --- a/cmake/libmultiprocess.cmake +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2025-present The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or https://opensource.org/license/mit/. - -function(add_libmultiprocess subdir) - # Set BUILD_TESTING to match BUILD_TESTS. BUILD_TESTING is a standard cmake - # option that controls whether enable_testing() is called, but in the bitcoin - # build a BUILD_TESTS option is used instead. - set(BUILD_TESTING "${BUILD_TESTS}") - add_subdirectory(${subdir} EXCLUDE_FROM_ALL) - # Apply core_interface compile options to libmultiprocess runtime library. - target_link_libraries(multiprocess PUBLIC $) - target_link_libraries(mputil PUBLIC $) - target_link_libraries(mpgen PUBLIC $) - # Mark capproto options as advanced to hide by default from cmake UI - mark_as_advanced(CapnProto_DIR) - mark_as_advanced(CapnProto_capnpc_IMPORTED_LOCATION) - mark_as_advanced(CapnProto_capnp_IMPORTED_LOCATION) - mark_as_advanced(CapnProto_capnp-json_IMPORTED_LOCATION) - mark_as_advanced(CapnProto_capnp-rpc_IMPORTED_LOCATION) - mark_as_advanced(CapnProto_capnp-websocket_IMPORTED_LOCATION) - mark_as_advanced(CapnProto_kj-async_IMPORTED_LOCATION) - mark_as_advanced(CapnProto_kj-gzip_IMPORTED_LOCATION) - mark_as_advanced(CapnProto_kj-http_IMPORTED_LOCATION) - mark_as_advanced(CapnProto_kj_IMPORTED_LOCATION) - mark_as_advanced(CapnProto_kj-test_IMPORTED_LOCATION) - mark_as_advanced(CapnProto_kj-tls_IMPORTED_LOCATION) - if(BUILD_TESTS) - # Add tests to "all" target so ctest can run them - set_target_properties(mptest PROPERTIES EXCLUDE_FROM_ALL OFF) - endif() - # Exclude examples from compilation database, because the examples are not - # built by default, and they contain generated c++ code. Without this - # exclusion, tools like clang-tidy and IWYU that make use of compilation - # database would complain that the generated c++ source files do not exist. An - # alternate fix could build "mpexamples" by default like "mptests" above. - set_target_properties(mpcalculator mpprinter mpexample PROPERTIES EXPORT_COMPILE_COMMANDS OFF) -endfunction() diff --git a/depends/packages/native_libmultiprocess.mk b/depends/packages/native_libmultiprocess.mk deleted file mode 100644 index f494f6e8a580..000000000000 --- a/depends/packages/native_libmultiprocess.mk +++ /dev/null @@ -1,15 +0,0 @@ -package=native_libmultiprocess -$(package)_local_dir=../src/ipc/libmultiprocess -$(package)_dependencies=native_capnp - -define $(package)_config_cmds - $($(package)_cmake) . -endef - -define $(package)_build_cmds - $(MAKE) -endef - -define $(package)_stage_cmds - $(MAKE) DESTDIR=$($(package)_staging_dir) install-bin -endef diff --git a/depends/packages/packages.mk b/depends/packages/packages.mk index 7f8f35a895a6..580d27187333 100644 --- a/depends/packages/packages.mk +++ b/depends/packages/packages.mk @@ -3,6 +3,6 @@ packages:= boost_packages = boost ipc_packages = capnp -multiprocess_native_packages = native_libmultiprocess native_capnp +multiprocess_native_packages = native_capnp usdt_linux_packages=systemtap diff --git a/depends/toolchain.cmake.in b/depends/toolchain.cmake.in index 820d6cd8e925..863a6b370e31 100644 --- a/depends/toolchain.cmake.in +++ b/depends/toolchain.cmake.in @@ -136,6 +136,5 @@ else() set(USDT_INCLUDE_DIR "${CMAKE_CURRENT_LIST_DIR}/systemtap/include" CACHE PATH "") endif() -set(MPGEN_EXECUTABLE "${CMAKE_CURRENT_LIST_DIR}/native/bin/mpgen" CACHE FILEPATH "") set(CAPNP_EXECUTABLE "${CMAKE_CURRENT_LIST_DIR}/native/bin/capnp" CACHE FILEPATH "") set(CAPNPC_CXX_EXECUTABLE "${CMAKE_CURRENT_LIST_DIR}/native/bin/capnpc-c++" CACHE FILEPATH "") diff --git a/doc/dependencies.md b/doc/dependencies.md index 005b6d285b36..393a51907d73 100644 --- a/doc/dependencies.md +++ b/doc/dependencies.md @@ -24,7 +24,6 @@ Bitcoin Core requires one of the following compilers. | [Boost](../depends/packages/boost.mk) | [link](https://www.boost.org/users/download/) | [1.74.0](https://github.com/bitcoin/bitcoin/pull/34107) | | [Cap'n Proto](../depends/packages/native_capnp.mk) ([IPC](multiprocess.md)) | [link](https://capnproto.org/) | [0.7.1](https://github.com/bitcoin/bitcoin/pull/33241) | | CMake | [link](https://cmake.org/) | [3.22](https://github.com/bitcoin/bitcoin/pull/30454) | -| [libmultiprocess](../depends/packages/native_libmultiprocess.mk) ([IPC](multiprocess.md)) | [link](https://github.com/bitcoin-core/libmultiprocess/tags) | [v7.0-pre1](https://github.com/bitcoin/bitcoin/pull/33517) | ### Runtime @@ -40,4 +39,3 @@ Bitcoin Core requires one of the following compilers. | --- | --- | --- | | Python (scripts, tests) | [link](https://www.python.org) | [3.10](https://github.com/bitcoin/bitcoin/pull/30527) | | [systemtap](../depends/packages/systemtap.mk) ([tracing](tracing.md)) | [link](https://sourceware.org/systemtap/) | N/A | - diff --git a/doc/design/multiprocess.md b/doc/design/multiprocess.md index f87e9c02b867..a5b7396d1ded 100644 --- a/doc/design/multiprocess.md +++ b/doc/design/multiprocess.md @@ -40,32 +40,31 @@ This section describes the major components of the Inter-Process Communication ( compatibility with Cap'n Proto. ### Cap’n Proto Files in [`src/ipc/capnp/`](../../src/ipc/capnp/) -- Corresponding to each abstract class, there are `.capnp` files within the [`src/ipc/capnp/`](../../src/ipc/capnp/) directory. These files are used as input to the `mpgen` tool (described below) to generate C++ code. -- These Cap’n Proto files ([learn more about Cap'n Proto RPC](https://capnproto.org/rpc.html)) define the structure and format of messages that are exchanged over IPC. They serve as blueprints for generating C++ code that bridges the gap between high-level C++ interfaces and low-level socket communication. +- Corresponding to each abstract class, there are `.capnp` files within the [`src/ipc/capnp/`](../../src/ipc/capnp/) directory. These files are used as input to Cap'n Proto's C++ compiler to generate low-level RPC classes. +- These Cap’n Proto files ([learn more about Cap'n Proto RPC](https://capnproto.org/rpc.html)) define the structure and format of messages that are exchanged over IPC. The native bridge in [`src/ipc/capnp/protocol.cpp`](../../src/ipc/capnp/protocol.cpp) adapts those generated classes to the high-level C++ interfaces. -### The `mpgen` Code Generation Tool -- A central component of the IPC framework is the `mpgen` tool which is part of the [`libmultiprocess` project](https://github.com/bitcoin-core/libmultiprocess). This tool takes the `.capnp` files as input and generates C++ code. -- The generated code handles IPC communication, translating interface calls into socket reads and writes. +### Cap'n Proto Code Generation +- Cap'n Proto's `capnp` and `capnpc-c++` tools generate the C++ RPC types from the `.capnp` files at build time. +- Generated files are build outputs. Hand-written code in [`src/ipc/capnp/`](../../src/ipc/capnp/) owns the policy for mapping Bitcoin Core interfaces, cancellation handlers, callbacks, and domain types to those generated RPC types. -### C++ Client Subclasses in Generated Code -- In the generated code, we have C++ client subclasses that inherit from the abstract classes in [`src/interfaces/`](../../src/interfaces/). These subclasses are the workhorses of the IPC mechanism. +### C++ Client Subclasses +- The native IPC bridge provides C++ client subclasses that inherit from the abstract classes in [`src/interfaces/`](../../src/interfaces/). These subclasses are the workhorses of the IPC mechanism. - They implement all the methods of the interface, marshalling arguments into a structured format, sending them as requests to the IPC server via a UNIX socket, and handling the responses. - These subclasses effectively mask the complexity of IPC, presenting a familiar C++ interface to developers. -- Internally, the client subclasses generated by the `mpgen` tool wrap [client classes generated by Cap'n Proto](https://capnproto.org/cxxrpc.html#clients), and use them to send IPC requests. The Cap'n Proto client classes are low-level, with non-blocking methods that use asynchronous I/O and pass request and response objects, while mpgen client subclasses provide normal C++ methods that block while executing and convert between request/response objects and arguments/return values. +- Internally, the client subclasses wrap [client classes generated by Cap'n Proto](https://capnproto.org/cxxrpc.html#clients), and use them to send IPC requests. Synchronous interface methods block on the Cap'n Proto event loop, while async interface methods schedule RPC promises and return cancellation handlers before completion callbacks can run. -### C++ Server Classes in Generated Code -- On the server side, corresponding generated C++ classes receive IPC requests. These server classes are responsible for unmarshalling method arguments, invoking the corresponding methods in the local [`src/interfaces/`](../../src/interfaces/) objects, and creating the IPC response. -- The server classes ensure that return values (including output argument values and thrown exceptions) are marshalled and sent back to the client, completing the communication cycle. -- Internally, the server subclasses generated by the `mpgen` tool inherit from [server classes generated by Cap'n Proto](https://capnproto.org/cxxrpc.html#servers), and use them to process IPC requests. +### C++ Server Classes +- On the server side, hand-written server classes inherit from [server classes generated by Cap'n Proto](https://capnproto.org/cxxrpc.html#servers). These server classes are responsible for unmarshalling method arguments, invoking the corresponding methods in the local [`src/interfaces/`](../../src/interfaces/) objects, and creating the IPC response. +- The server classes ensure that return values and translated errors are marshalled and sent back to the client, completing the communication cycle. -### The `libmultiprocess` Runtime Library -- **Core Functionality**: The `libmultiprocess` runtime library's primary function is to instantiate the generated client and server classes as needed. -- **Bootstrapping IPC Connections**: It provides functions for starting new IPC connections, specifically binding generated client and server classes for an initial `interfaces::Init` interface (defined in [`src/interfaces/init.h`](../../src/interfaces/init.h)) to a UNIX socket. This initial interface has methods returning other interfaces that different Bitcoin Core modules use to communicate after the bootstrapping phase. -- **Asynchronous I/O and Thread Management**: The library is also responsible for managing I/O and threading. Particularly, it ensures that IPC requests never block each other and that new threads on either side of a connection can always make client calls. It also manages worker threads on the server side of calls, ensuring that calls from the same client thread always execute on the same server thread (to avoid locking issues and support nested callbacks). +### The Native Cap'n Proto Runtime +- **Core Functionality**: The native runtime in [`src/ipc/capnp/protocol.cpp`](../../src/ipc/capnp/protocol.cpp) instantiates client and server adapters for the generated Cap'n Proto capabilities. +- **Bootstrapping IPC Connections**: It starts new IPC connections by binding an initial `interfaces::Init` interface (defined in [`src/interfaces/init.h`](../../src/interfaces/init.h)) to a UNIX socket. This initial interface has methods returning other interfaces that different Bitcoin Core modules use to communicate after the bootstrapping phase. +- **Asynchronous I/O and Thread Management**: The runtime owns the Cap'n Proto event loop thread for each client connection, schedules asynchronous calls on that thread, and dispatches completion callbacks on a separate callback executor so interface callbacks do not re-enter before cancellation handlers are returned. -### Type Hooks in [`src/ipc/capnp/*-types.h`](../../src/ipc/capnp/) -- **Custom Type Conversions**: In [`src/ipc/capnp/*-types.h`](../../src/ipc/capnp/), function overloads of `libmultiprocess` C++ functions, `mp::CustomReadField`, `mp::CustomBuildField`, `mp::CustomReadMessage` and `mp::CustomBuildMessage`, are defined. These overloads are used for customizing the conversion of specific C++ types to and from Cap’n Proto types. -- **Handling Special Cases**: The `mpgen` tool and `libmultiprocess` library can convert most C++ types to and from Cap’n Proto types automatically, including interface types, primitive C++ types, standard C++ types like `std::vector`, `std::set`, `std::map`, `std::tuple`, and `std::function`, as well as simple C++ structs that consist of aforementioned types and whose fields correspond 1:1 with Cap’n Proto struct fields. For other types, `*-types.h` files provide custom code to convert between C++ and Cap’n Proto data representations. +### Type Conversion Helpers +- **Custom Type Conversions**: [`src/ipc/capnp/common-types.h`](../../src/ipc/capnp/common-types.h) provides serialization helpers for Bitcoin Core types that are sent as Cap'n Proto `Data` fields. +- **Handling Special Cases**: Interface capabilities, callbacks, cancellation watches, and option-presence semantics are mapped explicitly in the native bridge instead of being inferred by a separate code generator. ### Protocol-Agnostic IPC Code in [`src/ipc/`](../../src/ipc/) - **Broad Applicability**: Unlike the Cap’n Proto-specific code in [`src/ipc/capnp/`](../../src/ipc/capnp/), the code in the [`src/ipc/`](../../src/ipc/) directory is protocol-agnostic. This enables potential support for other protocols, such as gRPC or a custom protocol in the future. @@ -76,17 +75,16 @@ This section describes the major components of the Inter-Process Communication ( ```mermaid flowchart TD - capnpFile[ipc/capnp/chain.capnp] -->|Input to| mpgenTool([mpgen Tool]) - mpgenTool -->|Generates| proxyTypesH[ipc/capnp/chain.capnp.proxy-types.h] - mpgenTool --> proxyClientCpp[ipc/capnp/chain.capnp.proxy-client.c++] - mpgenTool --> proxyServerCpp[ipc/capnp/chain.capnp.proxy-server.c++] - proxyTypesH -.->|Includes| interfaces/chain.h - proxyClientCpp -.-> interfaces/chain.h - proxyServerCpp -.-> interfaces/chain.h + capnpFile[ipc/capnp/init.capnp] -->|Input to| capnpc([capnpc-c++]) + capnpc -->|Generates| capnpH[ipc/capnp/init.capnp.h] + capnpc --> capnpCpp[ipc/capnp/init.capnp.c++] + capnpH -.-> protocolCpp[ipc/capnp/protocol.cpp] + capnpCpp -.-> protocolCpp + protocolCpp -.-> interfaces/init.h ``` -Diagram showing generated source files and includes. +Diagram showing generated Cap'n Proto files and the native bridge. ## Design Considerations @@ -102,7 +100,11 @@ The IPC mechanism is deliberately isolated from the rest of the codebase so less Building Bitcoin Core with IPC support is optional. The build system also ensures Cap’n Proto library headers can only be used within the [`src/ipc/capnp/`](../../src/ipc/capnp/) directory, not in other parts of the codebase. -The libmultiprocess runtime is designed to place as few constraints as possible on IPC interfaces and to make IPC calls act like normal function calls. Method arguments, return values, and exceptions are automatically serialized and sent between processes. Object references and `std::function` arguments are tracked to allow invoked code to call back into invoking code at any time. And there is a 1:1 threading model where every client thread has a corresponding server thread responsible for executing incoming calls from that thread (there can be multiple calls from the same thread due to callbacks) without blocking, and holding the same thread-local variables and locks so behavior is the same whether IPC is used or not. +The native runtime is designed to keep IPC concerns inside `src/ipc/`. The +high-level interfaces expose ordinary C++ values, capabilities, callbacks, and +cancellation handlers. The Cap'n Proto bridge translates those to generated RPC +requests and promises, while the node layer remains independent of Cap'n Proto +headers. ### Interface Definition Maintenance @@ -120,7 +122,12 @@ The currently defined IPC interfaces are unstable, and can change freely with no ## Security Considerations -The integration of [Cap’n Proto](https://capnproto.org/) and [libmultiprocess](https://github.com/bitcoin-core/libmultiprocess) into the Bitcoin Core architecture increases its potential attack surface. Cap’n Proto, being a complex and substantial new dependency, introduces potential sources of vulnerability, particularly through the creation of new UNIX sockets. The inclusion of libmultiprocess, while a smaller external dependency, also contributes to this risk. However, plans are underway to incorporate libmultiprocess as a git subtree, aligning it more closely with the project's well-reviewed internal libraries. While adopting these multiprocess features does introduce some risk, it's worth noting that they can be disabled, allowing builds without these new dependencies. This flexibility ensures that users can balance functionality with security considerations as needed. +The integration of [Cap’n Proto](https://capnproto.org/) into the Bitcoin Core +architecture increases its potential attack surface. Cap’n Proto introduces +potential sources of vulnerability, particularly through the creation of new UNIX +sockets and bidirectional RPC capabilities. While adopting these multiprocess +features does introduce some risk, IPC support can be disabled so users and +builders can balance functionality with security considerations as needed. ## Example Use Cases and Flows @@ -157,20 +164,19 @@ sequenceDiagram - The client process calls the `getBlockHash` method on a `Chain` object. This method is defined as a virtual method in [`src/interfaces/chain.h`](../../src/interfaces/chain.h). 2. **Translation to Cap’n Proto RPC** - - The `Chain::getBlockHash` virtual method is overridden by the `Chain` [client subclass](#c-client-subclasses-in-generated-code) to translate the method call into a Cap’n Proto RPC call. - - The client subclass is automatically generated by the `mpgen` tool from the [`chain.capnp`](https://github.com/ryanofsky/bitcoin/blob/pr/ipc/src/ipc/capnp/chain.capnp) file in [`src/ipc/capnp/`](../../src/ipc/capnp/). + - The `Chain::getBlockHash` virtual method is overridden by the `Chain` [client subclass](#c-client-subclasses) to translate the method call into a Cap’n Proto RPC call. + - The client subclass is implemented in the native bridge under [`src/ipc/capnp/`](../../src/ipc/capnp/) and uses Cap'n Proto generated client classes. 3. **Request Preparation and Dispatch** - - The `getBlockHash` method of the generated `Chain` client subclass in the client process populates a Cap’n Proto request with the `height` parameter, sends it to `bitcoin-node` process, and waits for a response. + - The `getBlockHash` method of the native `Chain` client subclass in the client process populates a Cap’n Proto request with the `height` parameter, sends it to `bitcoin-node` process, and waits for a response. 4. **Handling in bitcoin-node** - - Upon receiving the request, the Cap'n Proto dispatching code in the `bitcoin-node` process calls the `getBlockHash` method of the `Chain` [server class](#c-server-classes-in-generated-code). - - The server class is automatically generated by the `mpgen` tool from the [`chain.capnp`](https://github.com/ryanofsky/bitcoin/blob/pr/ipc/src/ipc/capnp/chain.capnp) file in [`src/ipc/capnp/`](../../src/ipc/capnp/). - - The `getBlockHash` method of the generated `Chain` server subclass in `bitcoin-node` receives a Cap’n Proto request object with the `height` parameter, and calls the `getBlockHash` method on its local `Chain` object with the provided `height`. + - Upon receiving the request, the Cap'n Proto dispatching code in the `bitcoin-node` process calls the `getBlockHash` method of the `Chain` [server class](#c-server-classes). + - The native server class in [`src/ipc/capnp/`](../../src/ipc/capnp/) receives a Cap’n Proto request object with the `height` parameter, and calls the `getBlockHash` method on its local `Chain` object with the provided `height`. - When the call returns, it encapsulates the return value in a Cap’n Proto response, which it sends back to the client process. 5. **Response and Return** - - The `getBlockHash` method of the generated `Chain` client subclass in the client process which sent the request now receives the response. + - The `getBlockHash` method of the native `Chain` client subclass in the client process which sent the request now receives the response. - It extracts the block hash value from the response, and returns it to the original caller. ## Future Enhancements @@ -181,7 +187,7 @@ Further improvements are possible such as: - Automatically generating `.capnp` files from C++ interface definitions (see [Interface Definition Maintenance](#interface-definition-maintenance)). - Simplifying and stabilizing interfaces (see [Interface Stability](#interface-stability)). - Adding sandbox features, restricting subprocess access to resources and data (see [https://eklitzke.org/multiprocess-bitcoin](https://eklitzke.org/multiprocess-bitcoin)). -- Using Cap'n Proto's support for [other languages](https://capnproto.org/otherlang.html), such as [Rust](https://github.com/capnproto/capnproto-rust), to allow code written in other languages to call Bitcoin Core C++ code, and vice versa (see [How to rustify libmultiprocess? #56](https://github.com/bitcoin-core/libmultiprocess/issues/56)). +- Using Cap'n Proto's support for [other languages](https://capnproto.org/otherlang.html), such as [Rust](https://github.com/capnproto/capnproto-rust), to allow code written in other languages to call Bitcoin Core C++ code. ## Conclusion @@ -201,23 +207,19 @@ This modularization represents an advancement in Bitcoin Core's architecture, of - **Cap’n Proto struct**: A structured data format used in Cap’n Proto, similar to structs in C++, for organizing and transporting data across different processes. -- **client class (in generated code)**: A C++ class generated from a Cap’n Proto interface which inherits from a Bitcoin Core abstract class, and implements each virtual method to send IPC requests to another process. (see also [components section](#c-client-subclasses-in-generated-code)) +- **client class**: A C++ class in the native bridge which inherits from a Bitcoin Core abstract class, and implements each virtual method to send IPC requests to another process. (see also [components section](#c-client-subclasses)) - **IPC (inter-process communication)**: Mechanisms that enable processes to exchange requests and data. - **ipc::Exception class**: A class within Bitcoin Core's protocol-agnostic IPC code that is thrown by client class methods when there is an IPC error. -- **libmultiprocess**: A custom library and code generation tool used for creating IPC interfaces and managing IPC connections. - - **marshalling**: Transforming an object’s memory representation for transmission. -- **mpgen tool**: A tool within the `libmultiprocess` suite that generates C++ code from Cap’n Proto files, facilitating IPC. - - **protocol-agnostic code**: Generic IPC code in [`src/ipc/`](../../src/ipc/) that does not rely on Cap’n Proto and could be used with other protocols. Distinct from code in [`src/ipc/capnp/`](../../src/ipc/capnp/) which relies on Cap’n Proto. - **RPC (remote procedure call)**: A protocol that enables a program to request a service from another program in a different address space or network. Bitcoin Core uses [JSON-RPC](https://en.wikipedia.org/wiki/JSON-RPC) for RPC. -- **server class (in generated code)**: A C++ class generated from a Cap’n Proto interface which handles requests sent by a _client class_ in another process. The request handled by calling a local Bitcoin Core interface method, and the return values (if any) are sent back in a response. (see also: [components section](#c-server-classes-in-generated-code)) +- **server class**: A C++ class in the native bridge which handles requests sent by a _client class_ in another process. The request is handled by calling a local Bitcoin Core interface method, and the return values (if any) are sent back in a response. (see also: [components section](#c-server-classes)) - **unix socket**: Communication endpoint which is a filesystem path, used for exchanging data between processes running on the same host. @@ -226,8 +228,7 @@ This modularization represents an advancement in Bitcoin Core's architecture, of ## References - **Cap’n Proto RPC protocol description**: https://capnproto.org/rpc.html -- **libmultiprocess project page**: https://github.com/bitcoin-core/libmultiprocess ## Acknowledgements -This design doc was written by @ryanofsky, who is grateful to all the reviewers who gave feedback and tested [multiprocess PRs](https://github.com/bitcoin/bitcoin/pull/28722), and everyone else who's helped with this project. Particular thanks to @ariard who deeply reviewed IPC code and improved the design of the IPC library and initialization process. @jnewbery who championed the early refactoring PRs and helped guide them through development and review. @sjors who has reviewed and repeatedly tested multiprocess code, reporting many issues and helping debug them. @hebasto, @fanquake, and @maflcko who made significant improvements to the build system and fixed countless build issues. @vasild and @jamesob who were brave contributors to the libmultiprocess library. And Chaincode Labs for making this work possible. Also thanks to ChatGPT, who actually wrote most of this document (not @ryanofsky). +This design doc was written by @ryanofsky, who is grateful to all the reviewers who gave feedback and tested [multiprocess PRs](https://github.com/bitcoin/bitcoin/pull/28722), and everyone else who's helped with this project. Particular thanks to @ariard who deeply reviewed IPC code and improved the design of the IPC library and initialization process. @jnewbery who championed the early refactoring PRs and helped guide them through development and review. @sjors who has reviewed and repeatedly tested multiprocess code, reporting many issues and helping debug them. @hebasto, @fanquake, and @maflcko who made significant improvements to the build system and fixed countless build issues. And Chaincode Labs for making this work possible. Also thanks to ChatGPT, who actually wrote most of this document (not @ryanofsky). diff --git a/doc/developer-notes.md b/doc/developer-notes.md index f3bbf6add1a3..2d8c3611b361 100644 --- a/doc/developer-notes.md +++ b/doc/developer-notes.md @@ -1139,19 +1139,6 @@ to check a subtree directory for consistency with its upstream repository. The tool instructions also include a list of the subtrees managed by Bitcoin Core. -To fully verify or update a subtree, add it as a remote: - -```sh -git remote add libmultiprocess https://github.com/bitcoin-core/libmultiprocess.git -``` - -To update the subtree: - -```sh -git fetch libmultiprocess -git subtree pull --prefix=src/ipc/libmultiprocess libmultiprocess master --squash -``` - The ultimate upstream of the few externally managed subtrees are: - src/leveldb @@ -1410,7 +1397,6 @@ there are [`interfaces::Chain`](../src/interfaces/chain.h), used to access the node's latest chain state, [`interfaces::Node`](../src/interfaces/node.h), used to control the node, and [`interfaces::Mining`](../src/interfaces/mining.h), ->>>>>>> acdcd09526 (nuke: wallet + psbt + external-signer (src, build, deps, ci, tests, docs)) used by RPC to generate block templates. There are also more specialized interface types like [`interfaces::Handler`](../src/interfaces/handler.h) passed to and from various interface methods. diff --git a/doc/multiprocess.md b/doc/multiprocess.md index 4d599f678107..64168624fc28 100644 --- a/doc/multiprocess.md +++ b/doc/multiprocess.md @@ -26,11 +26,10 @@ build/bin/bitcoin -m node -regtest -printtoconsole -debug=ipc ### Cross-compiling -When cross-compiling and not using depends, native code generation tools from [libmultiprocess](https://github.com/bitcoin-core/libmultiprocess) and [Cap'n Proto](https://capnproto.org/) are required. They can be passed to the cmake build by specifying `-DMPGEN_EXECUTABLE=/path/to/mpgen -DCAPNP_EXECUTABLE=/path/to/capnp -DCAPNPC_CXX_EXECUTABLE=/path/to/capnpc-c++` options. - -### External libmultiprocess installation - -By default the libmultiprocess sources at [../src/ipc/libmultiprocess/](../src/ipc/libmultiprocess/) are built as part of the bitcoin cmake build, but alternately an external [libmultiprocess](https://github.com/bitcoin-core/libmultiprocess/) cmake package can be used instead by following its [installation instructions](https://github.com/bitcoin-core/libmultiprocess/blob/master/doc/install.md) and specifying `-DWITH_EXTERNAL_LIBMULTIPROCESS=ON` to the bitcoin build, so it will use the external package instead of the sources. This can be useful when making changes to the upstream project. If libmultiprocess is not installed in a default system location it is possible to specify the [`CMAKE_PREFIX_PATH`](https://cmake.org/cmake/help/latest/envvar/CMAKE_PREFIX_PATH.html) environment variable to point to the installation prefix where libmultiprocess is installed. +When cross-compiling and not using depends, native code generation tools from +[Cap'n Proto](https://capnproto.org/) are required. They can be passed to the +cmake build by specifying `-DCAPNP_EXECUTABLE=/path/to/capnp +-DCAPNPC_CXX_EXECUTABLE=/path/to/capnpc-c++` options. ## Usage diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f5feb9959341..c1bdab596d77 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -19,10 +19,6 @@ include(../cmake/crc32c.cmake) include(../cmake/leveldb.cmake) include(../cmake/minisketch.cmake) add_subdirectory(univalue) -if (NOT WITH_EXTERNAL_LIBMULTIPROCESS) - include(../cmake/libmultiprocess.cmake) - add_libmultiprocess(ipc/libmultiprocess) -endif() include(../cmake/secp256k1.cmake) add_secp256k1(secp256k1) diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 191357fa6621..13113833345e 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -26,7 +26,7 @@ using node::BlockCreateOptions; static void AssembleBlock(benchmark::Bench& bench) { - const auto test_setup = MakeNoLogFileContext(); + auto test_setup = MakeNoLogFileContext(); CScriptWitness witness; witness.stack.push_back(WITNESS_STACK_ELEM_OP_TRUE); diff --git a/src/bitcoin.cpp b/src/bitcoin.cpp index 4b36a03a1406..4da4f35dd748 100644 --- a/src/bitcoin.cpp +++ b/src/bitcoin.cpp @@ -8,14 +8,14 @@ #include #include #include -#include +#include #include +#include #include #include #include #include -#include #include const TranslateFn G_TRANSLATION_FUN{nullptr}; @@ -23,7 +23,7 @@ const TranslateFn G_TRANSLATION_FUN{nullptr}; static constexpr auto HELP_USAGE = R"(Usage: %s [OPTIONS] COMMAND... Options: - -m, --multiprocess Run multiprocess binary bitcoin-node. + -m, --ipc-server Run IPC server binary bitcoin-node. -M, --monolithic Run monolithic binary bitcoind. (Default behavior) -v, --version Show version information -h, --help Show full help message @@ -47,7 +47,7 @@ Run '%s help' to see additional commands (e.g. for testing and debugging). )"; struct CommandLine { - std::optional use_multiprocess; + std::optional use_ipc_server; bool show_version{false}; bool show_help{false}; std::string_view command; @@ -118,10 +118,10 @@ CommandLine ParseCommandLine(int argc, char* argv[]) std::string_view arg = argv[i]; if (!cmd.command.empty()) { cmd.args.emplace_back(argv[i]); - } else if (arg == "-m" || arg == "--multiprocess") { - cmd.use_multiprocess = true; + } else if (arg == "-m" || arg == "--ipc-server") { + cmd.use_ipc_server = true; } else if (arg == "-M" || arg == "--monolithic") { - cmd.use_multiprocess = false; + cmd.use_ipc_server = false; } else if (arg == "-v" || arg == "--version") { cmd.show_version = true; } else if (arg == "-h" || arg == "--help" || arg == "help") { @@ -139,7 +139,7 @@ bool UseMultiprocess(const CommandLine& cmd) { // If -m or -M options were explicitly specified, there is no need to // further parse arguments to determine which to use. - if (cmd.use_multiprocess) return *cmd.use_multiprocess; + if (cmd.use_ipc_server) return *cmd.use_ipc_server; ArgsManager args; args.SetDefaultFlags(ArgsManager::ALLOW_ANY); @@ -154,8 +154,8 @@ bool UseMultiprocess(const CommandLine& cmd) } args.SelectConfigNetwork(args.GetChainTypeString()); - // If any -ipc* options are set these need to be processed by a - // multiprocess-capable binary. + // If any -ipc* options are set these need to be processed by the + // IPC-capable server binary. return args.IsArgSet("-ipcbind") || args.IsArgSet("-ipcconnect") || args.IsArgSet("-ipcfd"); } @@ -187,7 +187,7 @@ static void ExecCommand(const std::vector& args, std::string_view w auto try_exec = [&](fs::path exe_path, bool allow_notfound = true) { std::string exe_path_str{fs::PathToString(exe_path)}; exec_args[0] = exe_path_str.c_str(); - if (util::ExecVp(exec_args[0], (char*const*)exec_args.data()) == -1) { + if (util::ExecVp(exec_args[0], (char* const*)exec_args.data()) == -1) { if (allow_notfound && errno == ENOENT) return false; throw std::system_error(errno, std::system_category(), strprintf("execvp failed to execute '%s'", exec_args[0])); } @@ -217,11 +217,11 @@ static void ExecCommand(const std::vector& args, std::string_view w // in libexec/ (wrapper_dir.filename() == "bin" && try_exec(wrapper_dir.parent_path() / "libexec" / arg0.filename())) || #ifdef WIN32 - // Otherwise check the "daemon" subdirectory in a windows install. - (!wrapper_dir.empty() && try_exec(wrapper_dir / "daemon" / arg0.filename())) || + // Otherwise check the "daemon" subdirectory in a windows install. + (!wrapper_dir.empty() && try_exec(wrapper_dir / "daemon" / arg0.filename())) || #endif - // Otherwise look for target executable next to current wrapper - (!wrapper_dir.empty() && try_exec(wrapper_dir / arg0.filename(), fallback_os_search)) || - // Otherwise just look on the system path. - (fallback_os_search && try_exec(arg0.filename(), false)); + // Otherwise look for target executable next to current wrapper + (!wrapper_dir.empty() && try_exec(wrapper_dir / arg0.filename(), fallback_os_search)) || + // Otherwise just look on the system path. + (fallback_os_search && try_exec(arg0.filename(), false)); } diff --git a/src/init.cpp b/src/init.cpp index 19cbc46c53ef..67dbd06d0d77 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -7,8 +7,6 @@ #include -#include - #include #include #include @@ -16,24 +14,25 @@ #include #include #include +#include #include #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include #include #include #include #include #include #include -#include #include #include #include +#include #include #include #include @@ -66,6 +65,7 @@ #include #include #include +#include #include #include