[master] Zil 5417 create a driver program to run zilliqa code base native for testing purposes

build.sh

@@ -75,9 +75,9 @@ run_clang_tidy_fix=0
 run_code_coverage=0
 build_type="RelWithDebInfo"
 
-./scripts/license_checker.sh
-./scripts/ci_xml_checker.sh constants.xml
-./scripts/ci_xml_checker.sh constants_local.xml
+#./scripts/license_checker.sh
+#./scripts/ci_xml_checker.sh constants.xml
+#./scripts/ci_xml_checker.sh constants_local.xml
 if [ "$OS" != "osx" ]; then ./scripts/depends/check_guard.sh; fi
 
 # Find the git tag if we can and include it so we can report it in our GetVersion call

constants.xml

@@ -361,7 +361,7 @@
         <zilliqa>
             <METRIC_ZILLIQA_HOSTNAME>localhost</METRIC_ZILLIQA_HOSTNAME>
             <METRIC_ZILLIQA_PORT>8090</METRIC_ZILLIQA_PORT>
-            <METRIC_ZILLIQA_PROVIDER>PROMETHEUS</METRIC_ZILLIQA_PROVIDER>
+            <METRIC_ZILLIQA_PROVIDER>NONE</METRIC_ZILLIQA_PROVIDER>
             <METRIC_ZILLIQA_READER_EXPORT_MS>1000</METRIC_ZILLIQA_READER_EXPORT_MS>
             <METRIC_ZILLIQA_READER_TIMEOUT_MS>500</METRIC_ZILLIQA_READER_TIMEOUT_MS>
             <METRIC_ZILLIQA_SCHEMA>https://opentelemetry.io/schemas/1.2.0</METRIC_ZILLIQA_SCHEMA>

docker/Dockerfile

@@ -18,7 +18,7 @@ ARG SCILLA_VERSION=v0.13.4
 ARG SCILLA_IMAGE=zilliqa/scilla:${SCILLA_VERSION}
 
 # Common dependencies of the builder and runner stages.
-FROM ubuntu:22.04 AS base
+FROM arm64v8/ubuntu:22.04 AS base
 # Format guideline: one package per line and keep them alphabetically sorted
 RUN apt-get update \
     && apt-get install -y software-properties-common \
@@ -78,12 +78,11 @@ ENV SCCACHE_DIR="/sccache"
 ENV RUSTC_WRAPPER="/root/.cargo/bin/sccache"
 
 ARG CMAKE_VERSION=3.25.1
-RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.sh \
-    && echo "6598da34f0e3a0f763809e25cfdd646aa1d5e4d133c4277821e63ae5cfe09457 cmake-${CMAKE_VERSION}-Linux-x86_64.sh" | sha256sum -c \ 
+RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-aarch64.sh \
     && mkdir -p "${HOME}"/.local \
-    && bash ./cmake-${CMAKE_VERSION}-Linux-x86_64.sh --skip-license --prefix="${HOME}"/.local/ \
+    && bash ./cmake-${CMAKE_VERSION}-Linux-aarch64.sh --skip-license --prefix="${HOME}"/.local/ \
     && "${HOME}"/.local/bin/cmake --version \
-    && rm cmake-${CMAKE_VERSION}-Linux-x86_64.sh
+    && rm cmake-${CMAKE_VERSION}-Linux-aarch64.sh
 ENV PATH="/root/.local/bin:${PATH}"
 
 # Setup ccache
@@ -98,6 +97,7 @@ ARG VCPKG_ROOT=/vcpkg
 
 RUN ccache -p && ccache -z
 
+ENV VCPKG_FORCE_SYSTEM_BINARIES=1
 # If COMMIT_OR_TAG is a branch name or a tag, clone a shallow copy which is
 # faster; if this fails, just clone the full repo and checkout the commit.
 RUN git clone https://github.com/microsoft/vcpkg ${VCPKG_ROOT} \
@@ -116,9 +116,11 @@ RUN apt update -y \
   && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - \
   && apt update -y && apt install -y google-cloud-cli
 
-ENV VCPKG_BINARY_SOURCES="default,readwrite;x-gcs,gs://vcpkg/ubuntu/22.04/x64-linux-dynamic/${VCPKG_COMMIT_OR_TAG}/,read"
+ENV VCPKG_BINARY_SOURCES="default,readwrite;x-gcs,gs://vcpkg/ubuntu/22.04/arm64-linux-dynamic/${VCPKG_COMMIT_OR_TAG}/,readwrite"
 
-RUN --mount=type=cache,target=/root/.cache/vcpkg/ ${VCPKG_ROOT}/vcpkg install --triplet=x64-linux-dynamic
+RUN --mount=type=cache,target=/root/.cache/vcpkg/ \
+  echo "set(VCPKG_TARGET_ARCHITECTURE arm64)\nset(VCPKG_CRT_LINKAGE dynamic)\nset(VCPKG_LIBRARY_LINKAGE dynamic)\nset(VCPKG_CMAKE_SYSTEM_NAME Linux)\nset(VCPKG_FIXUP_ELF_RPATH ON)" >> ${VCPKG_ROOT}/triplets/community/arm64-linux-dynamic.cmake \
+  && ${VCPKG_ROOT}/vcpkg install --triplet=arm64-linux-dynamic
 
 ENV INSTALL_DIR=/usr/local
 ENV CCACHE_DIR="/ccache"
@@ -128,13 +130,13 @@ WORKDIR /zilliqa
 COPY src/libPersistence/*.proto src/libPersistence/
 COPY src/libUtils/*.proto src/libUtils/
 COPY evm-ds/ evm-ds
-RUN --mount=type=cache,target=/sccache/ cd evm-ds && PATH=/zilliqa/build/vcpkg_installed/x64-linux-dynamic/tools/protobuf:$PATH cargo build --release --package evm-ds && sccache --show-stats
+RUN --mount=type=cache,target=/sccache/ cd evm-ds && PATH=/zilliqa/build/vcpkg_installed/arm64-linux-dynamic/tools/protobuf:$PATH cargo build --release --package evm-ds && sccache --show-stats
 
 RUN ls -l /zilliqa/evm-ds/target
 
-RUN cp /zilliqa/build/vcpkg_installed/x64-linux-dynamic/lib/libffi.so /usr/local/lib  \
-  && cp /zilliqa/build/vcpkg_installed/x64-linux-dynamic/lib/libssl.so* /usr/local/lib  \
-  && cp /zilliqa/build/vcpkg_installed/x64-linux-dynamic/lib/libcrypto.so* /usr/local/lib
+RUN cp /zilliqa/build/vcpkg_installed/arm64-linux-dynamic/lib/libffi.so /usr/local/lib  \
+  && cp /zilliqa/build/vcpkg_installed/arm64-linux-dynamic/lib/libssl.so* /usr/local/lib  \
+  && cp /zilliqa/build/vcpkg_installed/arm64-linux-dynamic/lib/libcrypto.so* /usr/local/lib
 
 RUN apt-get install -y patchelf
 
@@ -191,8 +193,8 @@ RUN mkdir -p /scilla/0/bin2/ && cp -L /scilla/0/bin/* /scilla/0/bin2/ && strip /
 
 FROM builder AS test_runner
 RUN mkdir -p \
-    /scilla/0/bin /scilla/0/src/stdlib /scilla/0/vcpkg_installed/x64-linux-dynamic \
-    && ln -s /usr/local/lib /scilla/0/vcpkg_installed/x64-linux-dynamic
+    /scilla/0/bin /scilla/0/src/stdlib /scilla/0/vcpkg_installed/arm64-linux-dynamic \
+    && ln -s /usr/local/lib /scilla/0/vcpkg_installed/arm64-linux-dynamic
 # pour in scilla binaries
 COPY --from=scilla  /scilla/0/bin2                /scilla/0/bin
 # pour in scilla tests
@@ -224,7 +226,7 @@ COPY --from=builder /usr/local/lib/*.so* /usr/local/lib/
 COPY --from=builder /zilliqa/evm-ds/target/release/evm-ds /usr/local/bin/
 COPY --from=builder /zilliqa/evm-ds/log4rs.yml /usr/local/etc/
 
-ADD https://github.com/krallin/tini/releases/latest/download/tini /tini
+ADD https://github.com/krallin/tini/releases/latest/download/tini-arm64 /tini
 
 # The above COPY doesn't preserve symlinks which causes gcc/g++/cc/c++
 # to hang as stand-alone binaries on Ubuntu 22.04.

native/README.md

@@ -0,0 +1,54 @@
+# native
+
+## Ziliiqa Native Driver
+
+## Description
+
+Create a program that will generate the required environment to launch a native version of the Zilliqa blockchain that will run on a local or server machine environment.
+
+The Zilliqa binaries will be built using the build.sh script and will support native debugging at the source code level using local development tools of your choice.
+
+Initial version is to run on a single host but can be extended later to run across any hosts if required.
+
+The initial version is independent of any other Zilliqa technologies and has no dependency on any other Zilliqa toolsets.
+
+The driver supports both the constants.xml configuration format and can utilize additional contract accounts as defined in isolated-server-accounts.json
+
+To run the tool simply run the native.py with the following arguments or variations of:
+
+python3 native.py -n 6 -d 5 -l 1--port 30303 --websocket= --transaction-sender=0
+--ds-guard=4
+--shard-guard=0
+--bucket=zilliqa-devnet
+--origin-server=/<yourdir>/dev/native/tmp
+--multiplier-fanout=1
+--out-dir=/<yourdir>/dev/native/tmp
+--build-dir=/<yourdir>/Zilliqa/build
+--isolated-server-accounts
+
+## A simple startup script
+
+Startall.sh is provided as a first cut for launching the initial version
+
+
+## Prerequisites.
+
+Create a read/write directory /etc/zilliqa on your local machine (Annoying but hardcoded entry in zilliqa source)
+
+Out-dir is the path where the filesystem with binaries will be created, you need to run a web server there.   python3 -m http.server
+
+Currently you need to run a localstack instance.
+
+You can install localstack directly on your machine and start with localstack start
+
+
+## TODO
+
+Test, test, test
+
+Fixup the hard-coded path in the start.sh command, i usually run it inside by dir, I hope this is the last remaining defect with paths.
+
+run python web server under docker with the rest of the support utilities such as localstacck
+and the metrics daemons.
+
+Add the code required to add new join nodes and build new directories for them.

native/native.py

@@ -0,0 +1,139 @@
+#!/usr/bin/env python3
+# Copyright (C) 2023 Zilliqa
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+import argparse
+import os
+import shutil
+
+from natlib.data import Data
+from natlib.lib import (is_seedprv, \
+                        generate_nodes, str2fanout, str2uints, str2lookup)
+
+LOOKUP_TYPES = ('lookup', 'seedpub', 'seedprv')
+
+
+def main():
+    parser = argparse.ArgumentParser(description='initialize zilliqa')
+
+    parser.add_argument('--pod-name', help='pod name (TESTNET-TYPE-INDEX)')
+    parser.add_argument('--type', choices=['normal', 'lookup', 'new', 'seedprv'],
+                        help='node type (normal, lookup, new, seedprv)')
+    parser.add_argument('--index', type=int, help='node index')
+    parser.add_argument('--metadata-url', help='the base URL for metadata')
+    parser.add_argument('-n', type=int, help='number of all nodes', required=True)
+    parser.add_argument('-d', type=int, help='number of ds nodes', required=True)
+    parser.add_argument('-l', type=int, help='number of lookup nodes', required=True)
+    parser.add_argument('--ds-guard', type=int, default=0, help='number of ds guards')
+    parser.add_argument('--shard-guard', type=int, default=0, help='number of shard guards')
+    parser.add_argument('--out-dir', help='The output directory')
+    parser.add_argument('--build-dir', help='The build directory')
+    parser.add_argument('--block-number-size', default=64, type=int, choices=[64, 256],
+                        help='block number size (64-bit, 256-bit)')
+    # TODO: --seed-multiplier is redudant when --multiplier-fanout is used
+    parser.add_argument('--seed-multiplier', action='store_true', default=True, help='Support seed multiplier')
+    parser.add_argument('--multiplier-fanout', type=str2fanout, help='the multiplier fanouts')
+    parser.add_argument('--max-recovery', default='2', help='Max tries for recovering zilliqa node')
+    parser.add_argument('--resume', action='store_true', help='Resume from persistent storage')
+    parser.add_argument('--recover-from-testnet', action='store_true', help='Recover all nodes from persistent storage')
+    parser.add_argument('--transaction-sender', default='0', type=str2uints, metavar='N',
+                        help='List of lookup indices that send testing transactions')
+    parser.add_argument('--origin-server', help='set external origin server instead using in-cluster one')
+    parser.add_argument('--lookup-dns-domain', help='the DNS name for lookup, multiplier, and seedpub nodes')
+    parser.add_argument('--log-path', help='Set customized log path')
+    parser.add_argument('--bucket', help='Set bucket name')
+    parser.add_argument('--hosted-zone-id', help='hosted zone ID')
+    parser.add_argument('--txnsbackup', action='store_true', help='Enable storing txns backup to S3')
+    parser.add_argument('--websocket', type=str2lookup, metavar='TYPE1,TYPE2',
+                        help='enable websocket for lookup server of TYPE, can be any of {}'.format(LOOKUP_TYPES))
+
+    group0 = parser.add_argument_group('Host Network Mode')
+    group0.add_argument('--port', type=int, help='port for zilliqa application')
+
+    group1 = parser.add_argument_group('Local Testing Mode', 'for testing and debugging init.py')
+    group1.add_argument('--testing', action='store_true', help='enable local testing mode')
+
+    group2 = parser.add_argument_group('Private keys cleanup Mode',
+                                       'for cleaning up non-self private keys before running the working container')
+    group2.add_argument('--cleanup-private-keys', action='store_true', help='clean up non-self private keys')
+
+    group3 = parser.add_argument_group('Native mode settings', 'for defining how native will behave')
+    group3.add_argument('--isolated-server-accounts', action='store_true',
+                        help='use the isolated server accounts file from zilliqa directory')
+    group4 =parser.add_argument_group('Native mode settings', 'for defining how native will behave')
+    group4.add_argument('--daemon', default=False, help='If we should use the ZilliqaDaemon or lanuch zilliqa directly')
+
+    args = parser.parse_args()
+
+    # Inititalise the data object
+    data = Data()
+
+    # The origin server stores the network information
+    if args.origin_server is not None:
+        # if --origin-server is set, use it first
+        data.origin_server = args.origin_server
+    elif args.metadata_url is not None:
+        # metadata dns record used to be created with nginx ingress installation, not now
+        # If --metadata-url is set, use the internal one of origin
+        data.origin_server = '{}/origin/{}'.format(args.metadata_url, args.testnet)
+        # origin_server = args.metadata_url
+    else:
+        # This will not be used in the native implementation
+        data.origin_server = ""
+        print("We currently cannot start without something in origin server")
+        return 0
+
+    if args.isolated_server_accounts is not None:
+        # Copy into configmap
+        try:
+            shutil.copyfile("../isolated-server-accounts.json",
+                            os.path.join("/etc/zilliqa", "isolated-server-accounts.json"))
+        except:
+            print("Failed to copy isolated-server-accounts.json")
+            print("you need to create a /etc/zilliqa directory with rw access ")
+            return 0
+
+
+    if not data.get_ips_list_from_pseudo_origin(args):
+        print("Generating data from the command line arguments failed")
+        return 0
+    else:
+        data.print_details()
+
+    args.restart = False
+
+    try:
+        os.mkdir(args.out_dir)
+    except FileExistsError:
+        pass
+
+    cwd = os.getcwd()
+
+    generate_nodes(args, data, 'lookup', 0, args.l)
+    os.chdir(cwd)
+    generate_nodes(args, data, 'dsguard', 0, args.ds_guard)
+    os.chdir(cwd)
+    generate_nodes(args, data, 'normal', 0, args.d - args.ds_guard)
+    os.chdir(cwd)
+    generate_nodes(args, data, 'multiplier', 0, len(data.get_multiplier()))
+    os.chdir(cwd)
+    generate_nodes(args, data, 'seedpub', 0, sum(args.multiplier_fanout))
+
+    return 0
+
+
+# Press the green button in the gutter to run the script.
+if __name__ == '__main__':
+    main()

native/natlib/__init__.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+# Copyright (C) 2023 Zilliqa
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.