diff --git a/.dockerignore b/.dockerignore index c7f83c2c5b..9743482e93 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,53 @@ -target -.git +# Ignore some files that are not needed inside docker images, so that editing them doesn't cause +# docker to rebuild everything. +.dockerignore +.gitignore + +###################################### +# Below goes the contents of gitignore + +# Generated by Cargo +# will have compiled files and executables +**/target/ +# These are backup files generated by rustfmt +**/*.rs.bk + +.DS_Store + +# Intellij IDEA +.idea/ +*.iml +/customSpec.json + +# VSCode +.vscode/ + +#exclude python env +env/ + +# Test Python cache +test/**/__pycache__ + +# Files generated for the testing system +test/config.ini + +# The cache for docker container dependency +.cargo + +# The cache for chain data in container +.local + +# direnv cache +.direnv + +# Python compiled files +*.pyc + +# wasm +wasm-wrappers/pkg/ + +# 'mintlayer-data' will be mapped to home directories of docker containers, so everything +# inside it will be generated by the containers. +build-tools/docker/example-mainnet/mintlayer-data/* +# Same for example-mainnet-dns-server. +build-tools/docker/example-mainnet-dns-server/mintlayer-data/* diff --git a/.gitignore b/.gitignore index 3633284bdf..2d18d72413 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,9 @@ test/config.ini # wasm wasm-wrappers/pkg/ + +# 'mintlayer-data' will be mapped to home directories of docker containers, so everything +# inside it will be generated by the containers. +build-tools/docker/example-mainnet/mintlayer-data/* +# Same for example-mainnet-dns-server. +build-tools/docker/example-mainnet-dns-server/mintlayer-data/* diff --git a/api-server/scanner-daemon/src/config.rs b/api-server/scanner-daemon/src/config.rs index 2542efbfbe..b7df2889a1 100644 --- a/api-server/scanner-daemon/src/config.rs +++ b/api-server/scanner-daemon/src/config.rs @@ -28,19 +28,19 @@ pub struct ApiServerScannerArgs { /// Optional RPC address #[clap(long)] - pub rpc_address: Option, + pub node_rpc_address: Option, /// Path to the RPC cookie file. If not set, the value is read from the default cookie file location. #[clap(long)] - pub rpc_cookie_file: Option, + pub node_rpc_cookie_file: Option, /// RPC username (either provide a username and password, or use a cookie file. You cannot use both) #[clap(long)] - pub rpc_username: Option, + pub node_rpc_username: Option, /// RPC password (either provide a username and password, or use a cookie file. You cannot use both) #[clap(long)] - pub rpc_password: Option, + pub node_rpc_password: Option, /// Postgres config values #[clap(flatten)] diff --git a/api-server/scanner-daemon/src/main.rs b/api-server/scanner-daemon/src/main.rs index 178197c0d4..ba1ecf5e7d 100644 --- a/api-server/scanner-daemon/src/main.rs +++ b/api-server/scanner-daemon/src/main.rs @@ -27,7 +27,6 @@ use clap::Parser; use common::chain::{config::ChainType, ChainConfig}; use config::ApiServerScannerArgs; use node_comm::{make_rpc_client, rpc_client::NodeRpcClient}; -use node_lib::default_rpc_config; use rpc::RpcAuthData; use utils::{cookie::COOKIE_FILENAME, default_data_dir::default_data_dir_for_chain}; mod config; @@ -146,17 +145,17 @@ async fn main() -> Result<(), ApiServerScannerError> { let ApiServerScannerArgs { network, - rpc_address, - rpc_cookie_file, - rpc_username, - rpc_password, + node_rpc_address, + node_rpc_cookie_file, + node_rpc_username, + node_rpc_password, postgres_config, } = args; let chain_type: ChainType = network.into(); let chain_config = Arc::new(common::chain::config::Builder::new(chain_type).build()); - let rpc_auth = match (rpc_cookie_file, rpc_username, rpc_password) { + let node_rpc_auth = match (node_rpc_cookie_file, node_rpc_username, node_rpc_password) { (None, None, None) => { let cookie_file_path = default_data_dir_for_chain(chain_type.name()).join(COOKIE_FILENAME); @@ -173,14 +172,22 @@ async fn main() -> Result<(), ApiServerScannerError> { } }; - let default_rpc_bind_address = - || default_rpc_config(&chain_config).bind_address.expect("Can't fail").into(); + let default_node_rpc_bind_address = || { + node_lib::default_rpc_config(&chain_config) + .bind_address + .expect("Can't fail") + .into() + }; - let rpc_address = rpc_address.unwrap_or_else(default_rpc_bind_address); + let node_rpc_address = node_rpc_address.unwrap_or_else(default_node_rpc_bind_address); - let rpc_client = make_rpc_client(chain_config.clone(), rpc_address.to_string(), rpc_auth) - .await - .map_err(ApiServerScannerError::RpcError)?; + let rpc_client = make_rpc_client( + chain_config.clone(), + node_rpc_address.to_string(), + node_rpc_auth, + ) + .await + .map_err(ApiServerScannerError::RpcError)?; let storage = make_postgres_storage( postgres_config.postgres_host, diff --git a/build-tools/codecheck/codecheck.py b/build-tools/codecheck/codecheck.py index 0f14ff4ba3..43ef84aed9 100755 --- a/build-tools/codecheck/codecheck.py +++ b/build-tools/codecheck/codecheck.py @@ -29,6 +29,13 @@ r'|//' ] +COMMON_EXCLUDE_DIRS = [ + 'target', + '.git', + 'build-tools/docker/example-mainnet/mintlayer-data', + 'build-tools/docker/example-mainnet-dns-server/mintlayer-data' +] + # List Rust source files def rs_sources(exclude = []): @@ -47,7 +54,7 @@ def py_sources(exclude = []): # Cargo.toml files def cargo_toml_files(exclude = []): - exclude = [ os.path.normpath(dir) for dir in ['target', '.git', '.github'] + exclude ] + exclude = [ os.path.normpath(dir) for dir in COMMON_EXCLUDE_DIRS + ['.github'] + exclude ] is_excluded = lambda top, d: os.path.normpath(os.path.join(top, d).lower()) in exclude for top, dirs, files in os.walk('.', topdown=True): @@ -57,7 +64,7 @@ def cargo_toml_files(exclude = []): yield os.path.join(top, file) def _sources_with_extension(ext: str, exclude = []): - exclude = [ os.path.normpath(dir) for dir in ['target', '.git', '.github'] + exclude ] + exclude = [ os.path.normpath(dir) for dir in COMMON_EXCLUDE_DIRS + ['.github'] + exclude ] is_excluded = lambda top, d: os.path.normpath(os.path.join(top, d).lower()) in exclude for top, dirs, files in os.walk('.', topdown=True): @@ -73,7 +80,7 @@ def sources_with_extensions(exts: list[str], exclude = []): # All files def all_files(exclude = []): - exclude_full_paths = [ os.path.normpath(dir) for dir in ['target', '.git'] + exclude ] + exclude_full_paths = [ os.path.normpath(dir) for dir in COMMON_EXCLUDE_DIRS + exclude ] exclude_dir_names = ['__pycache__'] def is_excluded(top, d): diff --git a/build-tools/docker/Dockerfile.api-blockchain-scanner-daemon b/build-tools/docker/Dockerfile.api-blockchain-scanner-daemon new file mode 100644 index 0000000000..49fa09736d --- /dev/null +++ b/build-tools/docker/Dockerfile.api-blockchain-scanner-daemon @@ -0,0 +1,9 @@ +# Build Stage +FROM mintlayer-builder:latest AS builder + +# Runtime Stage +FROM mintlayer-runner-base + +COPY --from=builder /usr/src/target/release/api-blockchain-scanner-daemon /usr/bin + +CMD ["api-blockchain-scanner-daemon"] diff --git a/build-tools/docker/Dockerfile.api-web-server b/build-tools/docker/Dockerfile.api-web-server new file mode 100644 index 0000000000..1cfbdc7f9e --- /dev/null +++ b/build-tools/docker/Dockerfile.api-web-server @@ -0,0 +1,9 @@ +# Build Stage +FROM mintlayer-builder:latest AS builder + +# Runtime Stage +FROM mintlayer-runner-base + +COPY --from=builder /usr/src/target/release/api-web-server /usr/bin + +CMD ["api-web-server"] diff --git a/build-tools/docker/Dockerfile.builder b/build-tools/docker/Dockerfile.builder index aa5023f0fd..20c737c286 100644 --- a/build-tools/docker/Dockerfile.builder +++ b/build-tools/docker/Dockerfile.builder @@ -1,11 +1,25 @@ +# Note: this "source" stage only exists so that we could copy the source tree into +# the "builder" stage excluding the "build-tools" directory. This is to avoid rebuilding +# all the images every time when a file in "build-tools" is modified. Note that for most +# of the contents of "build-tools" this could be solved by adding them to .dockerignore, +# but there are files (e.g. entrypoint.sh) that are needed inside images, so they can't +# be ignored. +# +# TODO: dockerfile 1.7 syntax allows specifying --exclude for COPY, so the same can be done +# without an additional stage. But at the moment of writing this it's still experimental. +# Switch to using it when it becomes stable. +FROM rust as source +COPY . /src +RUN rm -r /src/build-tools + FROM rust AS builder WORKDIR /usr/src/ # Install necessary build dependencies for the GUI (such as X11, etc.) -RUN apt-get update && apt-get install -y ca-certificate && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/* -COPY . . +COPY --from=source /src/ /usr/src/ ARG NUM_JOBS=1 RUN cargo build --release -j${NUM_JOBS} diff --git a/build-tools/docker/Dockerfile.dns-server b/build-tools/docker/Dockerfile.dns-server new file mode 100644 index 0000000000..05b3f90ad8 --- /dev/null +++ b/build-tools/docker/Dockerfile.dns-server @@ -0,0 +1,9 @@ +# Build Stage +FROM mintlayer-builder:latest AS builder + +# Runtime Stage +FROM mintlayer-runner-base + +COPY --from=builder /usr/src/target/release/dns-server /usr/bin + +CMD ["dns-server"] diff --git a/build-tools/docker/Dockerfile.node-daemon b/build-tools/docker/Dockerfile.node-daemon index 60509d9b5f..4451057c62 100644 --- a/build-tools/docker/Dockerfile.node-daemon +++ b/build-tools/docker/Dockerfile.node-daemon @@ -1,16 +1,9 @@ # Build Stage FROM mintlayer-builder:latest AS builder - # Runtime Stage -FROM debian:bookworm-slim - -COPY --from=builder /usr/src/target/release/node-daemon /usr/bin - -# Node daemon listens on ports 13030 and 13031 -EXPOSE 13030 13031 +FROM mintlayer-runner-base -# Define mintlayer directory as a volume -VOLUME ["/root/.mintlayer"] +COPY --from=builder /usr/src/target/release/node-daemon /usr/bin/ CMD ["node-daemon"] diff --git a/build-tools/docker/Dockerfile.node-gui b/build-tools/docker/Dockerfile.node-gui deleted file mode 100644 index 05dd12c939..0000000000 --- a/build-tools/docker/Dockerfile.node-gui +++ /dev/null @@ -1,12 +0,0 @@ -# Build Stage -FROM mintlayer-builder:latest AS builder - -# Runtime Stage -FROM debian:bookworm-slim - -# Install necessary runtime dependencies for the GUI (such as X11, etc.) -RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/* - -COPY --from=builder /usr/src/target/release/node-gui /usr/bin/node-gui - -CMD ["node-gui"] diff --git a/build-tools/docker/Dockerfile.runner-base b/build-tools/docker/Dockerfile.runner-base new file mode 100644 index 0000000000..2512b9ba2f --- /dev/null +++ b/build-tools/docker/Dockerfile.runner-base @@ -0,0 +1,17 @@ +FROM debian:bookworm-slim + +RUN apt-get update && apt-get install -y gosu && rm -rf /var/lib/apt/lists/* + +WORKDIR /home/mintlayer + +# Define mintlayer directory as a volume; this will cause docker to create an anonymous +# volume for it if the user forgets to mount it explicitly. +VOLUME ["/home/mintlayer"] + +# Note: using an entrypoint script solves 2 problems: +# 1) We need to run the dockered app as a non-root user, for security purposes. +# 2) If the source of a bind mount doesn't exist when it's mounted, docker will create it +# automatically and the directory will be owned by root (if the host is Linux). The only way +# to change its ownership is to explicitly call 'chown' inside CMD or ENTRYPOINT. +COPY build-tools/docker/entrypoint.sh /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/build-tools/docker/Dockerfile.wallet-cli b/build-tools/docker/Dockerfile.wallet-cli index edf3da63b6..7d71397f28 100644 --- a/build-tools/docker/Dockerfile.wallet-cli +++ b/build-tools/docker/Dockerfile.wallet-cli @@ -2,9 +2,7 @@ FROM mintlayer-builder:latest AS builder # Runtime Stage -FROM debian:bookworm-slim - -RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/* +FROM mintlayer-runner-base COPY --from=builder /usr/src/target/release/wallet-cli /usr/bin/wallet-cli diff --git a/build-tools/docker/Dockerfile.wallet-rpc-daemon b/build-tools/docker/Dockerfile.wallet-rpc-daemon new file mode 100644 index 0000000000..25fb51829a --- /dev/null +++ b/build-tools/docker/Dockerfile.wallet-rpc-daemon @@ -0,0 +1,9 @@ +# Build Stage +FROM mintlayer-builder:latest AS builder + +# Runtime Stage +FROM mintlayer-runner-base + +COPY --from=builder /usr/src/target/release/wallet-rpc-daemon /usr/bin + +CMD ["wallet-rpc-daemon"] diff --git a/build-tools/docker/build.py b/build-tools/docker/build.py index a3b9fe60d5..e12804fad7 100644 --- a/build-tools/docker/build.py +++ b/build-tools/docker/build.py @@ -25,6 +25,9 @@ def build_docker_image(dockerfile_path, image_name, version, num_jobs=None): command = f"docker build -t {image_name}:{version} -f {dockerfile_path}" if num_jobs: command += f" --build-arg NUM_JOBS={num_jobs}" + # Note: "plain" output is more verbose, but it makes it easier to understand what went wrong + # when a problem occurs. + command += " --progress=plain" command += " ." try: @@ -73,23 +76,42 @@ def delete_docker_image(image_name, version): print(f"Failed to delete {full_image_name}.") -def build_instances(version, num_jobs=None): - build_docker_image("build-tools/docker/Dockerfile.builder", "mintlayer-builder", "latest", num_jobs) - build_docker_image("build-tools/docker/Dockerfile.node-daemon", "mintlayer/node-daemon", version) - build_docker_image("build-tools/docker/Dockerfile.node-gui", "mintlayer/node-gui", version) - build_docker_image("build-tools/docker/Dockerfile.wallet-cli", "mintlayer/wallet-cli", version) - delete_docker_image("mintlayer-builder", "latest") - - -def push_instances(version, latest): - push_docker_image("mintlayer/node-daemon",version , latest) - push_docker_image("mintlayer/node-gui",version , latest) - push_docker_image("mintlayer/wallet-cli",version , latest) +def build_instances(version, docker_hub_user, num_jobs): + if num_jobs is None: + num_jobs = os.cpu_count() or 1 + + build_docker_image("build-tools/docker/Dockerfile.builder", + "mintlayer-builder", "latest", num_jobs) + build_docker_image("build-tools/docker/Dockerfile.runner-base", + "mintlayer-runner-base", "latest", num_jobs) + build_docker_image("build-tools/docker/Dockerfile.node-daemon", + f"{docker_hub_user}/node-daemon", version) + build_docker_image("build-tools/docker/Dockerfile.api-blockchain-scanner-daemon", + f"{docker_hub_user}/api-blockchain-scanner-daemon", version) + build_docker_image("build-tools/docker/Dockerfile.api-web-server", + f"{docker_hub_user}/api-web-server", version) + build_docker_image("build-tools/docker/Dockerfile.wallet-cli", + f"{docker_hub_user}/wallet-cli", version) + build_docker_image("build-tools/docker/Dockerfile.wallet-rpc-daemon", + f"{docker_hub_user}/wallet-rpc-daemon", version) + build_docker_image("build-tools/docker/Dockerfile.dns-server", + f"{docker_hub_user}/dns-server", version) +# delete_docker_image("mintlayer-builder", "latest") + + +def push_instances(docker_hub_user, version, latest): + push_docker_image(f"{docker_hub_user}/node-daemon", version, latest) + push_docker_image(f"{docker_hub_user}/api-blockchain-scanner-daemon", version, latest) + push_docker_image(f"{docker_hub_user}/api-web-server", version, latest) + push_docker_image(f"{docker_hub_user}/wallet-cli", version, latest) + push_docker_image(f"{docker_hub_user}/wallet-rpc-daemon", version, latest) + push_docker_image(f"{docker_hub_user}/dns-server", version, latest) def main(): parser = argparse.ArgumentParser() parser.add_argument('--push', action='store_true', help='Push the Docker image to Docker Hub') + parser.add_argument('--docker-hub-user', help='Docker Hub username', default='mintlayer') parser.add_argument('--latest', action='store_true', help='Tag the Docker image as latest while pushing') parser.add_argument('--build', type=lambda x: (str(x).lower() == 'true'), default=True, help="Set to false avoid the build") parser.add_argument('--version', help='Override version number', default=None) @@ -99,12 +121,12 @@ def main(): version = args.version if args.version else get_cargo_version("Cargo.toml") if args.build: - build_instances(version, args.num_jobs) + build_instances(version, args.docker_hub_user, args.num_jobs) # Only push the image if the --push flag is provided if args.push: latest = args.latest - push_instances(version, latest) + push_instances(args.docker_hub_user, version, latest) if __name__ == "__main__": diff --git a/build-tools/docker/entrypoint.sh b/build-tools/docker/entrypoint.sh new file mode 100755 index 0000000000..c00017b081 --- /dev/null +++ b/build-tools/docker/entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -eu + +ML_USER_ID=${ML_USER_ID:-10001} +ML_GROUP_ID=${ML_GROUP_ID:-10001} + +# Create a group and a user; but first check if they already exist (in case an existing +# container is re-used). +if [[ ! $(id -g mintlayer 2> /dev/null) ]]; then + groupadd -g "$ML_GROUP_ID" mintlayer +fi +if [[ ! $(id -u mintlayer 2> /dev/null) ]]; then + # Note: /home/mintlayer should already be mounted to a host directory (or a volume), + # so it must exist. + useradd -u "$ML_USER_ID" -g mintlayer -d /home/mintlayer --no-create-home mintlayer +fi + +# Change the owner of /home/mintlayer (and therefore the host directory where it's mounted). +chown mintlayer:mintlayer /home/mintlayer + +# Launch the passed program using the "mintlayer" user. +# Note: +# 1) 'exec' will replace the current shell process with the specified program; this is needed +# for signals to be propagated corectly. +# 2) 'gosu' also uses the 'exec' syscall to launch the program (unlike 'su') and is needed +# for the same reason. +exec gosu mintlayer "$@" diff --git a/build-tools/docker/example-mainnet-dns-server/.env b/build-tools/docker/example-mainnet-dns-server/.env new file mode 100644 index 0000000000..9f4b149618 --- /dev/null +++ b/build-tools/docker/example-mainnet-dns-server/.env @@ -0,0 +1,22 @@ +# This will be used as a prefix for container and volume names +# (if not specified, `docker compose` will use the name of the parent directory as the project name). +COMPOSE_PROJECT_NAME=mintlayer-mainnet-dns-server + +# Dockerhub username, from which the docker images will be pulled. +DOCKERHUB_USERNAME=mintlayer +# The version of mintlayer binaries to use. +# This will be used together with DOCKERHUB_USERNAME to form full names of docker images +# (i.e. $DOCKERHUB_USERNAME/image_name:$ML_SOFTWARE_VERSION) +ML_SOFTWARE_VERSION=0.3.0 + +# The user and group ids that will be used to run the software. +ML_USER_ID=10001 +ML_GROUP_ID=10001 + +# The dns server will be available on this port. +DNS_SERVER_HOST_PORT=53 + +# Parameters for the dns-server. +DNS_SERVER_HOST_PARAM=seed.example.com +DNS_SERVER_NAMESERVER_PARAM=seed-ns.example.com +DNS_SERVER_MBOX_PARAM= diff --git a/build-tools/docker/example-mainnet-dns-server/README.md b/build-tools/docker/example-mainnet-dns-server/README.md new file mode 100644 index 0000000000..2caa55bf89 --- /dev/null +++ b/build-tools/docker/example-mainnet-dns-server/README.md @@ -0,0 +1,12 @@ +Here is an example `docker compose` project that runs the DNS server inside a Docker container. + +How to use: +---------- +1. Copy this directory to another location. Edit the `.env` file, specifying the required settings. + +2. `cd` to the copied project directory. + + To start the server, run `docker compose up`. + To shut it down, run `docket compose down`. + + (See `example-mainnet/README.md` for more info on what additional parameters can be specified for `docker compose` and how to make logging more verbose). diff --git a/build-tools/docker/example-mainnet-dns-server/docker-compose.yml b/build-tools/docker/example-mainnet-dns-server/docker-compose.yml new file mode 100644 index 0000000000..7b7d45421a --- /dev/null +++ b/build-tools/docker/example-mainnet-dns-server/docker-compose.yml @@ -0,0 +1,21 @@ +version: '3' + +services: + dns-server: + image: $DOCKERHUB_USERNAME/dns-server:$ML_SOFTWARE_VERSION + volumes: + - ./mintlayer-data:/home/mintlayer + command: dns-server + environment: + # Note: mainnet is the default network and 53 is the default port, so no need to specify them + # here; we do it mainly for demonstration purposes. + ML_DNS_SRV_NETWORK: mainnet + ML_DNS_SRV_BIND_ADDR: 0.0.0.0:53 + # Addresses taken from predefined_peer_addresses for mainnet. + ML_DNS_SRV_RESERVED_NODES: 51.159.232.144:3031,51.159.179.229:3031,151.115.35.206:3031,172.232.50.132:3031,103.3.61.21:3031 + ML_DNS_SRV_HOST: $DNS_SERVER_HOST_PARAM + ML_DNS_SRV_NAMESERVER: $DNS_SERVER_NAMESERVER_PARAM + ML_DNS_SRV_MBOX: $DNS_SERVER_MBOX_PARAM + RUST_LOG: + ports: + - "$DNS_SERVER_HOST_PORT:53/udp" diff --git a/build-tools/docker/example-mainnet/.env b/build-tools/docker/example-mainnet/.env new file mode 100644 index 0000000000..016ee9f598 --- /dev/null +++ b/build-tools/docker/example-mainnet/.env @@ -0,0 +1,55 @@ +# This will be used as a prefix for container and volume names +# (if not specified, `docker compose` will use the name of the parent directory as the project name). +COMPOSE_PROJECT_NAME=mintlayer-mainnet + +# Dockerhub username, from which the docker images will be pulled. +DOCKERHUB_USERNAME=mintlayer +# The version of mintlayer binaries to use. +# This will be used together with DOCKERHUB_USERNAME to form full names of docker images +# (i.e. $DOCKERHUB_USERNAME/image_name:$ML_SOFTWARE_VERSION) +ML_SOFTWARE_VERSION=0.3.0 + +# The user and group ids that will be used to run the software. +ML_USER_ID=10001 +ML_GROUP_ID=10001 + +# User name and db name for the postgres dbms used by the api server. Since the entire dbms will be +# run in a dedicated docker container using a dedicated docker volume, the names don't matter much. +# Just note that the db will be created when the volume is first initialized, so changing +# the name later won't work. +API_SERVER_POSTGRES_USER=postgres +API_SERVER_POSTGRES_DB=postgres +# The password for the postgres dbms. +API_SERVER_POSTGRES_PASSWORD=use-strong-password + +# The node's rpc port will be mapped to this port on the host machine. +# Note that it will only be reachable from localhost and not from other network interfaces. +NODE_RPC_HOST_PORT=3030 +# Username and password for node rpc calls. +NODE_RPC_USERNAME=username +NODE_RPC_PASSWORD=use-another-strong-password + +# The node's p2p port will be mapped to this port on the host machine. +# IMPORTANT: changing this port alone won't be enough to make your node discoverable by other nodes +# (or, if it was already discoverable, it may make it not discoverable anymore). +# The reason is that the node will advertise itself to the peers as "your_global_ip_address:the_port_you_specify_here". +# So, if you are behind NAT, you have to also make sure that "the_port_you_specify_here" is open to +# the outside world and mapped to "the_port_you_specify_here" on your host machine. +# On the other hand if you don't care about incoming connections from other nodes, just leave +# this variable as is. +NODE_P2P_HOST_PORT=3031 + +# Wallet rpc daemon's port will be mapped to this port on the host machine. +# This is reachable only from localhost. +WALLET_RPC_DAEMON_HOST_PORT=3034 +# Username and password for wallet rpc calls. +WALLET_RPC_DAEMON_USERNAME=username +WALLET_RPC_DAEMON_PASSWORD=use-yet-another-strong-password + +# Host machine's port to which api server's postgres port will be mapped. +# This is only useful if you want to examine the contents of the db yourself, e.g. via PgAdmin. +# Reachable only from localhost. +API_SERVER_POSTGRES_HOST_PORT=5434 + +# The API web server's port will be mapped to this port on the host machine. +API_SERVER_HOST_PORT=3000 diff --git a/build-tools/docker/example-mainnet/README.md b/build-tools/docker/example-mainnet/README.md new file mode 100644 index 0000000000..42a08fc6d5 --- /dev/null +++ b/build-tools/docker/example-mainnet/README.md @@ -0,0 +1,33 @@ +Here is an example `docker compose` project that runs various Mintlayer binaries +inside Docker containers. + +How to use: +---------- +1. Copy this directory to another location. Optionally, edit `docker-compose.yml`, +commenting out services that you won't need. Edit values in `.env` according to your needs. + + Then `cd` to the copied project directory. + + +2. To start the available services, run `docker compose up`. + + When the corresponding docker containers start, their `home` directories will be mounted + to the `mintlayer-data` subdirectory of the project directory. + + Note: this `mintlayer-data` directory will contain all the data produced by the containers + except for the api-server's database, which will be stored in a dedicated docker volume. + + Additionally, you can pass `-d` (or `--detach`) to `docker compose up` to run the containers + in detached mode. + To examine the logs in this case you can run `docker compose logs -f`. + + Note: the `RUST_LOG` environment variable is propagated into the containers, so if you need + more verbose logging, just set `RUST_LOG` on the host system, e.g. to `debug`. + + Finally, to stop and remove the containers, run `docker compose down`. + +3. It's also possible to run `wallet-cli` interactively in a docker container as well. + To do so, run the following after starting the services: + ``` + docker compose run --rm wallet-cli + ``` diff --git a/build-tools/docker/example-mainnet/docker-compose.yml b/build-tools/docker/example-mainnet/docker-compose.yml new file mode 100644 index 0000000000..2de8f13b90 --- /dev/null +++ b/build-tools/docker/example-mainnet/docker-compose.yml @@ -0,0 +1,137 @@ +version: '3' + +x-common: &ml-common + volumes: + - ./mintlayer-data:/home/mintlayer + +x-common-env: &ml-common-env + RUST_LOG: + ML_USER_ID: + ML_GROUP_ID: + +services: + node-daemon: + <<: *ml-common + image: $DOCKERHUB_USERNAME/node-daemon:$ML_SOFTWARE_VERSION + command: node-daemon mainnet + environment: + <<: *ml-common-env + # For rpc, the default rpc bind address is '127.0.0.1', so it'll only be reachable from the + # same container; to make it reachable from other containers or from the host machine, we + # have to set it to '0.0.0.0'. + ML_MAINNET_NODE_RPC_BIND_ADDRESS: 0.0.0.0:3030 + # For p2p, the default address is already '0.0.0.0'. But it's better to use the same port + # as the one exposed on the host system, otherwise other nodes won't be able to establish + # outbound connections to this node. + # Details: the node determines its own "public" p2p address by combining its ip address + # as it is seen by a peer with the port that it is listening on; this address is then + # advertised to other peers and propagated through the network. So, if the listening port + # differs from the one exposed on the host, the advertised address will be incorrect. + ML_MAINNET_NODE_P2P_BIND_ADDRESSES: 0.0.0.0:$NODE_P2P_HOST_PORT + # Rpc username and password. + ML_MAINNET_NODE_RPC_USERNAME: $NODE_RPC_USERNAME + ML_MAINNET_NODE_RPC_PASSWORD: $NODE_RPC_PASSWORD + ports: + # This is only needed if you want to access the node's rpc interface from the host system. + # Note that here we also set the ip address to bind to on the host to 127.0.0.1; because of + # this, rpc connections will only be allowed from the host machine and not from the "outside + # world". + - "127.0.0.1:$NODE_RPC_HOST_PORT:3030" + # This is only needed if you want the node to be able to accept incoming p2p connections + # from other nodes. + - "$NODE_P2P_HOST_PORT:$NODE_P2P_HOST_PORT" + + ### Services needed to run the api server ### + api-postgres-db: + image: postgres + restart: always + environment: + POSTGRES_USER: $API_SERVER_POSTGRES_USER + POSTGRES_PASSWORD: $API_SERVER_POSTGRES_PASSWORD + POSTGRES_DB: $API_SERVER_POSTGRES_DB + ports: + # This is only needed if you want to examine the contents of the db from the host system, + # e.g. via PgAdmin. + - "127.0.0.1:$API_SERVER_POSTGRES_HOST_PORT:5432" + volumes: + # Explicitly mount postgres docker image's mount point to a named volume (without this, + # docker will create an anonymous volume instead). + - api_postgres_db:/var/lib/postgresql/data + + api-blockchain-scanner-daemon: + <<: *ml-common + image: $DOCKERHUB_USERNAME/api-blockchain-scanner-daemon:$ML_SOFTWARE_VERSION + command: api-blockchain-scanner-daemon + depends_on: + - api-postgres-db + - node-daemon + environment: + <<: *ml-common-env + ML_API_SCANNER_DAEMON_POSTGRES_HOST: api-postgres-db + ML_API_SCANNER_DAEMON_POSTGRES_USER: $API_SERVER_POSTGRES_USER + ML_API_SCANNER_DAEMON_POSTGRES_PASSWORD: $API_SERVER_POSTGRES_PASSWORD + ML_API_SCANNER_DAEMON_POSTGRES_DATABASE: $API_SERVER_POSTGRES_DB + ML_API_SCANNER_DAEMON_NODE_RPC_ADDRESS: node-daemon:3030 + ML_API_SCANNER_DAEMON_NODE_RPC_USERNAME: $NODE_RPC_USERNAME + ML_API_SCANNER_DAEMON_NODE_RPC_PASSWORD: $NODE_RPC_PASSWORD + + api-web-server: + <<: *ml-common + image: $DOCKERHUB_USERNAME/api-web-server:$ML_SOFTWARE_VERSION + command: api-web-server + depends_on: + - api-postgres-db + - api-blockchain-scanner-daemon + - node-daemon + environment: + <<: *ml-common-env + ML_API_WEB_SRV_BIND_ADDRESS: 0.0.0.0:3000 + ML_API_WEB_SRV_POSTGRES_HOST: api-postgres-db + ML_API_WEB_SRV_POSTGRES_USER: $API_SERVER_POSTGRES_USER + ML_API_WEB_SRV_POSTGRES_PASSWORD: $API_SERVER_POSTGRES_PASSWORD + ML_API_WEB_SRV_POSTGRES_DATABASE: $API_SERVER_POSTGRES_DB + ML_API_WEB_SRV_NODE_RPC_ADDRESS: node-daemon:3030 + ML_API_WEB_SRV_NODE_RPC_USERNAME: $NODE_RPC_USERNAME + ML_API_WEB_SRV_NODE_RPC_PASSWORD: $NODE_RPC_PASSWORD + ports: + - "$API_SERVER_HOST_PORT:3000" + ### End of services needed to run the api server ### + + wallet-rpc-daemon: + <<: *ml-common + image: $DOCKERHUB_USERNAME/wallet-rpc-daemon:$ML_SOFTWARE_VERSION + command: wallet-rpc-daemon mainnet + depends_on: + - node-daemon + environment: + <<: *ml-common-env + ML_MAINNET_WALLET_RPC_DAEMON_NODE_RPC_ADDRESS: node-daemon:3030 + ML_MAINNET_WALLET_RPC_DAEMON_NODE_RPC_USERNAME: $NODE_RPC_USERNAME + ML_MAINNET_WALLET_RPC_DAEMON_NODE_RPC_PASSWORD: $NODE_RPC_PASSWORD + # Same as for the node, the default rpc bind address is '127.0.0.1' here; we need to set it + # to '0.0.0.0' to make it reachable from other containers/host machine. + ML_MAINNET_WALLET_RPC_DAEMON_RPC_BIND_ADDRESS: 0.0.0.0:3034 + ML_MAINNET_WALLET_RPC_DAEMON_RPC_USERNAME: $WALLET_RPC_DAEMON_USERNAME + ML_MAINNET_WALLET_RPC_DAEMON_RPC_PASSWORD: $WALLET_RPC_DAEMON_PASSWORD + ports: + - "127.0.0.1:$WALLET_RPC_DAEMON_HOST_PORT:3034" + + # wallet-cli is not a real service; we just need a service definition for it in order to be able + # to run it via "docker compose run" + wallet-cli: + <<: *ml-common + image: $DOCKERHUB_USERNAME/wallet-cli:$ML_SOFTWARE_VERSION + command: wallet-cli mainnet + depends_on: + - node-daemon + environment: + <<: *ml-common-env + ML_MAINNET_WALLET_NODE_RPC_ADDRESS: node-daemon:3030 + ML_MAINNET_WALLET_NODE_RPC_USERNAME: $NODE_RPC_USERNAME + ML_MAINNET_WALLET_NODE_RPC_PASSWORD: $NODE_RPC_PASSWORD + profiles: + # Put it in a separate profile, so that it's not started automatically by "docker compose up". + - wallet_cli + +volumes: + api_postgres_db: diff --git a/node-lib/src/options.rs b/node-lib/src/options.rs index 02ac1c6dd5..7ed12e2780 100644 --- a/node-lib/src/options.rs +++ b/node-lib/src/options.rs @@ -49,10 +49,13 @@ pub struct Options { #[derive(Subcommand, Clone, Debug)] pub enum Command { /// Run the mainnet node. + #[clap(mut_args(clap_utils::env_adder("MAINNET_NODE")))] Mainnet(RunOptions), /// Run the testnet node. + #[clap(mut_args(clap_utils::env_adder("TESTNET_NODE")))] Testnet(RunOptions), /// Run the regtest node. + #[clap(mut_args(clap_utils::env_adder("REGTEST_NODE")))] Regtest(Box), } diff --git a/p2p/src/peer_manager/mod.rs b/p2p/src/peer_manager/mod.rs index 0be2416bf5..3cff1b5cb8 100644 --- a/p2p/src/peer_manager/mod.rs +++ b/p2p/src/peer_manager/mod.rs @@ -317,6 +317,7 @@ where /// This won't work for majority of nodes but that should be accepted. fn discover_own_address( &mut self, + peer_id: PeerId, peer_role: PeerRole, common_services: Services, node_address_as_seen_by_peer: Option, @@ -366,7 +367,16 @@ where // Send only one address because of the rate limiter (see `ADDR_RATE_INITIAL_SIZE`). // Select a random address to give all addresses a chance to be discovered by the network. - discovered_own_addresses.into_iter().choose(&mut make_pseudo_rng()) + let chosen_discovered_address = + discovered_own_addresses.iter().choose(&mut make_pseudo_rng()).cloned(); + + log::debug!( + "Own addresses discovered for peer {peer_id}: {:?}, chosen address: {:?}", + discovered_own_addresses, + chosen_discovered_address + ); + + chosen_discovered_address } /// Send address announcement to the selected peer (if the address is new) @@ -947,6 +957,7 @@ where ); let discovered_own_address = self.discover_own_address( + peer_id, peer_role, info.common_services, node_address_as_seen_by_peer, diff --git a/wallet/wallet-cli-lib/src/config.rs b/wallet/wallet-cli-lib/src/config.rs index 095ee8899f..f886f95fa8 100644 --- a/wallet/wallet-cli-lib/src/config.rs +++ b/wallet/wallet-cli-lib/src/config.rs @@ -23,9 +23,13 @@ use utils_networking::NetworkAddressWithPort; #[derive(Subcommand, Clone, Debug)] pub enum Network { + #[clap(mut_args(clap_utils::env_adder("MAINNET_WALLET")))] Mainnet(CliArgs), + #[clap(mut_args(clap_utils::env_adder("TESTNET_WALLET")))] Testnet(CliArgs), + #[clap(mut_args(clap_utils::env_adder("REGTEST_WALLET")))] Regtest(Box), + #[clap(mut_args(clap_utils::env_adder("SIGNET_WALLET")))] Signet(CliArgs), } diff --git a/wallet/wallet-rpc-lib/src/cmdline.rs b/wallet/wallet-rpc-lib/src/cmdline.rs index 5d53c58519..6738f9801d 100644 --- a/wallet/wallet-rpc-lib/src/cmdline.rs +++ b/wallet/wallet-rpc-lib/src/cmdline.rs @@ -45,12 +45,15 @@ impl WalletRpcDaemonArgs { #[derive(clap::Subcommand)] pub enum WalletRpcDaemonCommand { /// Run the mainnet wallet. + #[clap(mut_args(clap_utils::env_adder("MAINNET_WALLET_RPC_DAEMON")))] Mainnet(WalletRpcDaemonChainArgs), /// Run the testnet wallet. + #[clap(mut_args(clap_utils::env_adder("TESTNET_WALLET_RPC_DAEMON")))] Testnet(WalletRpcDaemonChainArgs), /// Run the regtest wallet. + #[clap(mut_args(clap_utils::env_adder("REGTEST_WALLET_RPC_DAEMON")))] Regtest { #[command(flatten)] args: WalletRpcDaemonChainArgs, @@ -76,7 +79,6 @@ impl WalletRpcDaemonCommand { } #[derive(clap::Args)] -#[clap(mut_args(clap_utils::env_adder("WALLET_RPC_DAEMON")))] #[command( version, about, @@ -215,7 +217,7 @@ pub fn make_wallet_config( rpc_username: Option, rpc_password: Option, rpc_no_authentication: bool, - wallet_rpc_address: Option, + wallet_rpc_bind_address: Option, chain_type: ChainType, ) -> Result { let rpc_config = { @@ -230,7 +232,7 @@ pub fn make_wallet_config( _ => panic!("Should not happen due to arg constraints"), }; - let bind_addr = match wallet_rpc_address { + let bind_addr = match wallet_rpc_bind_address { None => { let port = WalletRpcConfig::default_port(chain_type); std::net::SocketAddr::new(std::net::Ipv4Addr::LOCALHOST.into(), port)