Update docker images and provide examples

.dockerignore

@@ -1,2 +1,53 @@
-target
-.git
+# Ignore some files that are not needed inside docker images, so that editing them doesn't cause
+# docker to rebuild everything.
+.dockerignore
+.gitignore
+
+######################################
+# Below goes the contents of gitignore
+
+# Generated by Cargo
+# will have compiled files and executables
+**/target/
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+.DS_Store
+
+# Intellij IDEA
+.idea/
+*.iml
+/customSpec.json
+
+# VSCode
+.vscode/
+
+#exclude python env
+env/
+
+# Test Python cache
+test/**/__pycache__
+
+# Files generated for the testing system
+test/config.ini
+
+# The cache for docker container dependency
+.cargo
+
+# The cache for chain data in container
+.local
+
+# direnv cache
+.direnv
+
+# Python compiled files
+*.pyc
+
+# wasm
+wasm-wrappers/pkg/
+
+# 'mintlayer-data' will be mapped to home directories of docker containers, so everything
+# inside it will be generated by the containers.
+build-tools/docker/example-mainnet/mintlayer-data/*
+# Same for example-mainnet-dns-server.
+build-tools/docker/example-mainnet-dns-server/mintlayer-data/*

.gitignore

@@ -37,3 +37,9 @@ test/config.ini
 
 # wasm
 wasm-wrappers/pkg/
+
+# 'mintlayer-data' will be mapped to home directories of docker containers, so everything
+# inside it will be generated by the containers.
+build-tools/docker/example-mainnet/mintlayer-data/*
+# Same for example-mainnet-dns-server.
+build-tools/docker/example-mainnet-dns-server/mintlayer-data/*

api-server/scanner-daemon/src/config.rs

@@ -28,19 +28,19 @@ pub struct ApiServerScannerArgs {
 
     /// Optional RPC address
     #[clap(long)]
-    pub rpc_address: Option<NetworkAddressWithPort>,
+    pub node_rpc_address: Option<NetworkAddressWithPort>,
 
     /// Path to the RPC cookie file. If not set, the value is read from the default cookie file location.
     #[clap(long)]
-    pub rpc_cookie_file: Option<String>,
+    pub node_rpc_cookie_file: Option<String>,
 
     /// RPC username (either provide a username and password, or use a cookie file. You cannot use both)
     #[clap(long)]
-    pub rpc_username: Option<String>,
+    pub node_rpc_username: Option<String>,
 
     /// RPC password (either provide a username and password, or use a cookie file. You cannot use both)
     #[clap(long)]
-    pub rpc_password: Option<String>,
+    pub node_rpc_password: Option<String>,
 
     /// Postgres config values
     #[clap(flatten)]

api-server/scanner-daemon/src/main.rs

@@ -27,7 +27,6 @@ use clap::Parser;
 use common::chain::{config::ChainType, ChainConfig};
 use config::ApiServerScannerArgs;
 use node_comm::{make_rpc_client, rpc_client::NodeRpcClient};
-use node_lib::default_rpc_config;
 use rpc::RpcAuthData;
 use utils::{cookie::COOKIE_FILENAME, default_data_dir::default_data_dir_for_chain};
 mod config;
@@ -146,17 +145,17 @@ async fn main() -> Result<(), ApiServerScannerError> {
 
     let ApiServerScannerArgs {
         network,
-        rpc_address,
-        rpc_cookie_file,
-        rpc_username,
-        rpc_password,
+        node_rpc_address,
+        node_rpc_cookie_file,
+        node_rpc_username,
+        node_rpc_password,
         postgres_config,
     } = args;
 
     let chain_type: ChainType = network.into();
     let chain_config = Arc::new(common::chain::config::Builder::new(chain_type).build());
 
-    let rpc_auth = match (rpc_cookie_file, rpc_username, rpc_password) {
+    let node_rpc_auth = match (node_rpc_cookie_file, node_rpc_username, node_rpc_password) {
         (None, None, None) => {
             let cookie_file_path =
                 default_data_dir_for_chain(chain_type.name()).join(COOKIE_FILENAME);
@@ -173,14 +172,22 @@ async fn main() -> Result<(), ApiServerScannerError> {
         }
     };
 
-    let default_rpc_bind_address =
-        || default_rpc_config(&chain_config).bind_address.expect("Can't fail").into();
+    let default_node_rpc_bind_address = || {
+        node_lib::default_rpc_config(&chain_config)
+            .bind_address
+            .expect("Can't fail")
+            .into()
+    };
 
-    let rpc_address = rpc_address.unwrap_or_else(default_rpc_bind_address);
+    let node_rpc_address = node_rpc_address.unwrap_or_else(default_node_rpc_bind_address);
 
-    let rpc_client = make_rpc_client(chain_config.clone(), rpc_address.to_string(), rpc_auth)
-        .await
-        .map_err(ApiServerScannerError::RpcError)?;
+    let rpc_client = make_rpc_client(
+        chain_config.clone(),
+        node_rpc_address.to_string(),
+        node_rpc_auth,
+    )
+    .await
+    .map_err(ApiServerScannerError::RpcError)?;
 
     let storage = make_postgres_storage(
         postgres_config.postgres_host,

build-tools/codecheck/codecheck.py

@@ -29,6 +29,13 @@
     r'|//'
 ]
 
+COMMON_EXCLUDE_DIRS = [
+    'target',
+    '.git',
+    'build-tools/docker/example-mainnet/mintlayer-data',
+    'build-tools/docker/example-mainnet-dns-server/mintlayer-data'
+]
+
 
 # List Rust source files
 def rs_sources(exclude = []):
@@ -47,7 +54,7 @@ def py_sources(exclude = []):
 
 # Cargo.toml files
 def cargo_toml_files(exclude = []):
-    exclude = [ os.path.normpath(dir) for dir in ['target', '.git', '.github'] + exclude ]
+    exclude = [ os.path.normpath(dir) for dir in COMMON_EXCLUDE_DIRS + ['.github'] + exclude ]
     is_excluded = lambda top, d: os.path.normpath(os.path.join(top, d).lower()) in exclude
 
     for top, dirs, files in os.walk('.', topdown=True):
@@ -57,7 +64,7 @@ def cargo_toml_files(exclude = []):
                 yield os.path.join(top, file)
 
 def _sources_with_extension(ext: str, exclude = []):
-    exclude = [ os.path.normpath(dir) for dir in ['target', '.git', '.github'] + exclude ]
+    exclude = [ os.path.normpath(dir) for dir in COMMON_EXCLUDE_DIRS + ['.github'] + exclude ]
     is_excluded = lambda top, d: os.path.normpath(os.path.join(top, d).lower()) in exclude
 
     for top, dirs, files in os.walk('.', topdown=True):
@@ -73,7 +80,7 @@ def sources_with_extensions(exts: list[str], exclude = []):
 
 # All files
 def all_files(exclude = []):
-    exclude_full_paths = [ os.path.normpath(dir) for dir in ['target', '.git'] + exclude ]
+    exclude_full_paths = [ os.path.normpath(dir) for dir in COMMON_EXCLUDE_DIRS + exclude ]
     exclude_dir_names = ['__pycache__']
 
     def is_excluded(top, d):

build-tools/docker/Dockerfile.api-blockchain-scanner-daemon

@@ -0,0 +1,9 @@
+# Build Stage
+FROM mintlayer-builder:latest AS builder
+
+# Runtime Stage
+FROM mintlayer-runner-base
+
+COPY --from=builder /usr/src/target/release/api-blockchain-scanner-daemon /usr/bin
+
+CMD ["api-blockchain-scanner-daemon"]

build-tools/docker/Dockerfile.api-web-server

@@ -0,0 +1,9 @@
+# Build Stage
+FROM mintlayer-builder:latest AS builder
+
+# Runtime Stage
+FROM mintlayer-runner-base
+
+COPY --from=builder /usr/src/target/release/api-web-server /usr/bin
+
+CMD ["api-web-server"]

build-tools/docker/Dockerfile.builder

@@ -1,11 +1,25 @@
+# Note: this "source" stage only exists so that we could copy the source tree into
+# the "builder" stage excluding the "build-tools" directory. This is to avoid rebuilding
+# all the images every time when a file in "build-tools" is modified. Note that for most
+# of the contents of "build-tools" this could be solved by adding them to .dockerignore,
+# but there are files (e.g. entrypoint.sh) that are needed inside images, so they can't
+# be ignored.
+#
+# TODO: dockerfile 1.7 syntax allows specifying --exclude for COPY, so the same can be done
+# without an additional stage. But at the moment of writing this it's still experimental.
+# Switch to using it when it becomes stable.
+FROM rust as source
+COPY . /src
+RUN rm -r /src/build-tools
+
 FROM rust AS builder
 
 WORKDIR /usr/src/
 
 # Install necessary build dependencies for the GUI (such as X11, etc.)
-RUN apt-get update && apt-get install -y ca-certificate && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/*
 
-COPY . .
+COPY --from=source /src/ /usr/src/
 
 ARG NUM_JOBS=1
 RUN cargo build --release -j${NUM_JOBS}

build-tools/docker/Dockerfile.dns-server

@@ -0,0 +1,9 @@
+# Build Stage
+FROM mintlayer-builder:latest AS builder
+
+# Runtime Stage
+FROM mintlayer-runner-base
+
+COPY --from=builder /usr/src/target/release/dns-server /usr/bin
+
+CMD ["dns-server"]

build-tools/docker/Dockerfile.node-daemon

@@ -1,16 +1,9 @@
 # Build Stage
 FROM mintlayer-builder:latest AS builder
 
-
 # Runtime Stage
-FROM debian:bookworm-slim
-
-COPY --from=builder /usr/src/target/release/node-daemon /usr/bin
-
-# Node daemon listens on ports 13030 and 13031
-EXPOSE 13030 13031
+FROM mintlayer-runner-base
 
-# Define mintlayer directory as a volume
-VOLUME ["/root/.mintlayer"]
+COPY --from=builder /usr/src/target/release/node-daemon /usr/bin/
 
 CMD ["node-daemon"]

build-tools/docker/Dockerfile.runner-base

@@ -0,0 +1,17 @@
+FROM debian:bookworm-slim
+
+RUN apt-get update && apt-get install -y gosu && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /home/mintlayer
+
+# Define mintlayer directory as a volume; this will cause docker to create an anonymous
+# volume for it if the user forgets to mount it explicitly.
+VOLUME ["/home/mintlayer"]
+
+# Note: using an entrypoint script solves 2 problems:
+# 1) We need to run the dockered app as a non-root user, for security purposes.
+# 2) If the source of a bind mount doesn't exist when it's mounted, docker will create it
+# automatically and the directory will be owned by root (if the host is Linux). The only way
+# to change its ownership is to explicitly call 'chown' inside CMD or ENTRYPOINT.
+COPY build-tools/docker/entrypoint.sh /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

build-tools/docker/Dockerfile.wallet-cli

@@ -2,9 +2,7 @@
 FROM mintlayer-builder:latest AS builder
 
 # Runtime Stage
-FROM debian:bookworm-slim
-
-RUN apt-get update && apt-get install -y ca-certificates && rm -rf /var/lib/apt/lists/*
+FROM mintlayer-runner-base
 
 COPY --from=builder /usr/src/target/release/wallet-cli /usr/bin/wallet-cli
 

build-tools/docker/Dockerfile.wallet-rpc-daemon

@@ -0,0 +1,9 @@
+# Build Stage
+FROM mintlayer-builder:latest AS builder
+
+# Runtime Stage
+FROM mintlayer-runner-base
+
+COPY --from=builder /usr/src/target/release/wallet-rpc-daemon /usr/bin
+
+CMD ["wallet-rpc-daemon"]

build-tools/docker/build.py

@@ -25,6 +25,9 @@ def build_docker_image(dockerfile_path, image_name, version, num_jobs=None):
     command = f"docker build -t {image_name}:{version} -f {dockerfile_path}"
     if num_jobs:
         command += f" --build-arg NUM_JOBS={num_jobs}"
+    # Note: "plain" output is more verbose, but it makes it easier to understand what went wrong
+    # when a problem occurs.
+    command += " --progress=plain"
     command += " ."
 
     try:
@@ -73,23 +76,42 @@ def delete_docker_image(image_name, version):
         print(f"Failed to delete {full_image_name}.")
 
 
-def build_instances(version, num_jobs=None):
-    build_docker_image("build-tools/docker/Dockerfile.builder", "mintlayer-builder", "latest", num_jobs)
-    build_docker_image("build-tools/docker/Dockerfile.node-daemon", "mintlayer/node-daemon", version)
-    build_docker_image("build-tools/docker/Dockerfile.node-gui", "mintlayer/node-gui", version)
-    build_docker_image("build-tools/docker/Dockerfile.wallet-cli", "mintlayer/wallet-cli", version)
-    delete_docker_image("mintlayer-builder", "latest")
-
-
-def push_instances(version, latest):
-    push_docker_image("mintlayer/node-daemon",version , latest)
-    push_docker_image("mintlayer/node-gui",version , latest)
-    push_docker_image("mintlayer/wallet-cli",version , latest)
+def build_instances(version, docker_hub_user, num_jobs):
+    if num_jobs is None:
+        num_jobs = os.cpu_count() or 1
+
+    build_docker_image("build-tools/docker/Dockerfile.builder",
+                        "mintlayer-builder", "latest", num_jobs)
+    build_docker_image("build-tools/docker/Dockerfile.runner-base",
+                        "mintlayer-runner-base", "latest", num_jobs)
+    build_docker_image("build-tools/docker/Dockerfile.node-daemon",
+                        f"{docker_hub_user}/node-daemon", version)
+    build_docker_image("build-tools/docker/Dockerfile.api-blockchain-scanner-daemon",
+                        f"{docker_hub_user}/api-blockchain-scanner-daemon", version)
+    build_docker_image("build-tools/docker/Dockerfile.api-web-server",
+                        f"{docker_hub_user}/api-web-server", version)
+    build_docker_image("build-tools/docker/Dockerfile.wallet-cli",
+                        f"{docker_hub_user}/wallet-cli", version)
+    build_docker_image("build-tools/docker/Dockerfile.wallet-rpc-daemon",
+                        f"{docker_hub_user}/wallet-rpc-daemon", version)
+    build_docker_image("build-tools/docker/Dockerfile.dns-server",
+                        f"{docker_hub_user}/dns-server", version)
+#    delete_docker_image("mintlayer-builder", "latest")
+
+
+def push_instances(docker_hub_user, version, latest):
+    push_docker_image(f"{docker_hub_user}/node-daemon", version, latest)
+    push_docker_image(f"{docker_hub_user}/api-blockchain-scanner-daemon", version, latest)
+    push_docker_image(f"{docker_hub_user}/api-web-server", version, latest)
+    push_docker_image(f"{docker_hub_user}/wallet-cli", version, latest)
+    push_docker_image(f"{docker_hub_user}/wallet-rpc-daemon", version, latest)
+    push_docker_image(f"{docker_hub_user}/dns-server", version, latest)
 
 
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('--push', action='store_true', help='Push the Docker image to Docker Hub')
+    parser.add_argument('--docker-hub-user', help='Docker Hub username', default='mintlayer')
     parser.add_argument('--latest', action='store_true', help='Tag the Docker image as latest while pushing')
     parser.add_argument('--build', type=lambda x: (str(x).lower() == 'true'), default=True, help="Set to false avoid the build")
     parser.add_argument('--version', help='Override version number', default=None)
@@ -99,12 +121,12 @@ def main():
     version = args.version if args.version else get_cargo_version("Cargo.toml")
 
     if args.build:
-        build_instances(version, args.num_jobs)
+        build_instances(version, args.docker_hub_user, args.num_jobs)
 
     # Only push the image if the --push flag is provided
     if args.push:
         latest = args.latest
-        push_instances(version, latest)
+        push_instances(args.docker_hub_user, version, latest)
 
 
 if __name__ == "__main__":