Merge pull request #124 from craigcomstock/ENT-12668

ENT-12668: Fix nt-discovery.sh usage and other improvements

Author: craigcomstock

.github/workflows/python-tests.yml renamed to .github/workflows/tests.yml

@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: Python tests
+name: Tests
 
 on:
   push:
@@ -10,7 +10,7 @@ on:
     branches: [ master ]
 
 jobs:
-  test:
+  python_version:
     runs-on: ubuntu-20.04
     env:
       # Temporary workaround for Python 3.5 failures - May 2024, see CFE-4395
@@ -48,7 +48,9 @@ jobs:
         pip install dist/cf_remote-*.whl
     - name: Sanity check
       run: cf-remote -V
-# No bash tests yet, see: https://github.com/cfengine/cfbs
-#    - name: Run bash tests
-#      run: |
-#        UNSAFE_TESTS=1 bash test/shell/all.sh
+    - name: Run docker test
+      run: |
+        bash tests/docker/0*.sh
+    - name: Run unsafe tests
+      run: |
+        bash tests/unsafe/0*.sh

MANIFEST.in

@@ -0,0 +1 @@
+include cf_remote/nt-discovery.sh

README.md

@@ -6,8 +6,9 @@ Commands for provisioning hosts in the cloud (AWS or GCP) are also available.
 
 ## Requirements
 
-cf-remote requires python 3.6 or greater.
-SSH must already be configured and running on the remote host.
+- cf-remote requires python 3.6 or greater.
+- SSH must be configured in such a way that cf-remote can login without a password.
+- An sftp server for transferring files on UNIX hosts. e.g. openssh-sftp-server for debian-based distributions.
 
 ## Installation
 
@@ -22,6 +23,10 @@ $ pip3 install cf-remote
 ### See information about remote host
 
 The `info` command can be used to check basic information about a system.
+The --hosts/-H option accepts [user@]hostname[:port] for the hostname.
+In the case that hostname is an ipv6 address use literal square brackets as described in RFC-3986 (https://www.ietf.org/rfc/rfc3986.txt)
+
+e.g. user@[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:8022
 
 ```
 $ cf-remote info -H 34.241.203.218

cf_remote/aramid.py

@@ -27,6 +27,7 @@
 from collections import namedtuple
 import subprocess
 import time
+from urllib.parse import urlparse
 
 DEFAULT_SSH_ARGS = [
     "-oLogLevel=ERROR",
@@ -130,6 +131,7 @@ def communicate(self, timeout=1, ignore_failed=False):
             raise _TaskError("Failed to communicate with the process") from e
         else:
             if self.proc.returncode == 255:  # SSH error
+                self.stderr += err
                 if self._retries > 0:
                     # wait for the rest of timeout (if any) and restart the process
                     time.sleep(max(timeout - (time.time() - start), 0))
@@ -152,7 +154,10 @@ def communicate(self, timeout=1, ignore_failed=False):
                             % (self.host.host_name, self._max_retries)
                         )
                     else:
-                        raise _TaskError("SSH failed on '%s'" % self.host.host_name)
+                        raise _TaskError(
+                            "SSH failed on '%s' with error '%s'"
+                            % (self.host.host_name, self.stderr)
+                        )
             else:
                 self.done = True
                 self.stdout += out
@@ -191,22 +196,20 @@ def get_result(self):
 class Host:
     """A remote host to execute commands on or copy files to"""
 
-    def __init__(self, host_name, user="root", extra_ssh_args=None):
+    def __init__(
+        self, host_name, user="root", port=_DEFAULT_SSH_PORT, extra_ssh_args=None
+    ):
         """
         :param str host_name: host name or IP of the host
         :param str user: user name to use to login to the host
         :param extra_ssh_args: extra SSH arguments to use when opening an SSH
                                connection to the host
 
         """
-        if ":" in host_name:
-            host_name, port = host_name.split(":")
-            self.host_name = host_name
-            self.port = int(port)
-        else:
-            self.host_name = host_name
-            self.port = _DEFAULT_SSH_PORT
-        self.user = user
+        parts = urlparse("ssh://%s" % host_name)
+        self.user = user or parts.username
+        self.port = port or parts.port
+        self.host_name = parts.hostname
         self.extra_ssh_args = extra_ssh_args or []
 
         self.tasks = []

nt-discovery.sh renamed to cf_remote/nt-discovery.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import sys
 import re
-from os.path import basename
+from os.path import basename, dirname, join, exists
 from collections import OrderedDict
 
 from cf_remote.utils import (
@@ -208,7 +208,16 @@ def get_info(host, *, users=None, connection=None):
     else:
         data["os"] = "unix"
 
-        scp("nt-discovery.sh", host, connection, hide=True)
+        cf_remote_dir = dirname(__file__)
+        script_path = join(cf_remote_dir, "nt-discovery.sh")
+        if not exists(script_path):
+            sys.exit("%s does not exist" % script_path)
+        scp(
+            script_path,
+            host,
+            connection,
+            hide=True,
+        )
         discovery = parse_envfile(ssh_sudo(connection, "bash nt-discovery.sh"))
 
         if discovery is None:
@@ -255,7 +264,6 @@ def get_info(host, *, users=None, connection=None):
 
 @auto_connect
 def install_package(host, pkg, data, *, connection=None):
-
     print("Installing: '{}' on '{}'".format(pkg, host))
     output = None
     if ".deb" in pkg:
@@ -478,7 +486,6 @@ def install_host(
     remote_download=False,
     trust_keys=None
 ):
-
     data = get_info(host, connection=connection)
     if show_info:
         print_info(data)

cf_remote/remote.py

@@ -4,6 +4,7 @@
 import shutil
 import signal
 import subprocess
+from urllib.parse import urlparse
 
 from cf_remote import aramid
 from cf_remote import log
@@ -30,22 +31,29 @@ def run(self, command, hide=False):
             stderr=subprocess.STDOUT,
             shell=True,
             universal_newlines=True,
+            cwd=os.environ["HOME"],
         )
         result.retcode = result.returncode
         return result
 
     def put(self, src, hide=False):
+        dst = os.path.join(os.environ["HOME"], os.path.basename(src))
         src = os.path.abspath(src)
-        dst = os.path.basename(src)
         if src != dst:
             if not hide:
                 print("Local copy: '%s' -> '%s'" % (src, dst))
             shutil.copy(src, dst)
 
 
 class Connection:
-    def __init__(self, host, user, connect_kwargs=None):
+    def __init__(self, host, user, connect_kwargs=None, port=aramid._DEFAULT_SSH_PORT):
+        log.debug(
+            "Initializing Connection: host '%s' user '%s' port '%s'"
+            % (host, user, port)
+        )
+
         self.ssh_host = host
+        self.ssh_port = port
         self.ssh_user = user
         self._connect_kwargs = connect_kwargs
 
@@ -56,16 +64,22 @@ def __init__(self, host, user, connect_kwargs=None):
             "ssh",
             "-M",
             "-N",
+            "-p %s" % self.ssh_port,
             "-oControlPath=%s" % self._control_path,
         ]
         control_master_args.extend(aramid.DEFAULT_SSH_ARGS)
-        control_master_args.append("%s@%s" % (user, host))
+        control_master_args.append("%s@%s" % (self.ssh_user, self.ssh_host))
 
+        log.debug(
+            "Attempting to open SSH Control Master process with command: %s"
+            % " ".join(control_master_args)
+        )
         self._ssh_control_master = subprocess.Popen(
-            control_master_args
+            control_master_args,
         )  # stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
 
         self.needs_sudo = self.run("echo $UID", hide=True).stdout.strip() != "0"
+        log.debug("Connection initialized")
 
     def __del__(self):
         # If we have an SSH Control Master running, signal it to terminate.
@@ -83,16 +97,17 @@ def run(self, command, hide=False):
         # If the Control Master process is running (poll() returns None), let's
         # reuse its connection.
         if self._ssh_control_master.poll() is None:
+            log.debug("Control Master is running, using it")
             extra_ssh_args.extend(["-oControlPath=%s" % self._control_path])
 
-        ahost = aramid.Host(self.ssh_host, self.ssh_user, extra_ssh_args)
+        ahost = aramid.Host(self.ssh_host, self.ssh_user, self.ssh_port, extra_ssh_args)
         results = aramid.execute([ahost], command, echo=(not hide))
         return results[ahost][0]
 
     def put(self, src, hide=False):
         dst = os.path.basename(src)
-        ahost = aramid.Host(self.ssh_host, self.ssh_user)
-        results = aramid.put([ahost], src, dst, echo=(not hide))
+        ahost = aramid.Host(self.ssh_host, self.ssh_user, self.ssh_port)
+        results = aramid.put([ahost], src, dst=dst, echo=(not hide))
         return results[ahost][0].retcode
 
     def __enter__(self, *args, **kwargs):
@@ -105,12 +120,11 @@ def __exit__(self, *args, **kwargs):
 def connect(host, users=None):
     log.debug("Connecting to '%s'" % host)
     log.debug("users= '%s'" % users)
-    if "@" in host:
-        parts = host.split("@")
-        assert len(parts) == 2
-        host = parts[1]
-        if not users:
-            users = [parts[0]]
+    parts = urlparse("ssh://%s" % host)
+    host = parts.hostname
+    if not users:
+        users = [parts.username]
+    port = parts.port or aramid._DEFAULT_SSH_PORT
     if not users:
         users = [
             "Administrator",
@@ -127,14 +141,17 @@ def connect(host, users=None):
             users = [whoami()] + users
     for user in users:
         try:
-            log.debug("Attempting ssh: %s@%s" % (user, host))
+            log.debug("Attempting ssh: %s@%s:%s" % (user, host, port))
             connect_kwargs = {}
             key = os.getenv("CF_REMOTE_SSH_KEY")
             if key:
                 connect_kwargs["key_filename"] = os.path.expanduser(key)
-            c = Connection(host=host, user=user, connect_kwargs=connect_kwargs)
+            c = Connection(
+                host=host, user=user, port=port, connect_kwargs=connect_kwargs
+            )
             c.ssh_user = user
             c.ssh_host = host
+            c.ssh_port = port
             c.run("whoami", hide=True)
             return c
         except aramid.ExecutionError:

cf_remote/ssh.py

@@ -10,7 +10,7 @@ trap cleanup EXIT
 
 # this is a fairly exhaustive test and will take some time
 # spawn all reasonable "platform" specifications
-function test() {
+function test_spawn() {
   platform=$1
   version=$2
 
@@ -46,17 +46,18 @@ for platform in debian rhel windows debian-9 ubuntu-22 centos-7 rhel-9 windows-2
   cleanup
 done
 for version in 9 10 11 12; do
-  test debian $version
+  test_spawn debian $version
 done
 for version in 7 8; do
-  test centos $version
+  test_spawn centos $version
 done
 for version in 7 8 9; do
-  test rhel $version
+  test_spawn rhel $version
 done
 for version in 2008 2012 2016 2019 2022; do
-  test windows $version
+  test_spawn windows $version
 done
 for version in 16-04 18-04 20-04 22-04; do
-  test ubuntu "$version"
+  test_spawn ubuntu "$version"
 done
+test_spawn alpine

tests/aws-spawn-test.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -ex
+set -o pipefail
+
+rm -f log
+error () {
+  echo "=== error occurred, rc=$?, logs follow ==="
+  [ -f log ] && cat log
+}
+trap error ERR
+
+dir=$(dirname "$0")
+name=cf-remote-debian-test-host
+
+docker stop "$name" || true
+docker rm "$name" || true
+docker build -t "$name" "$dir" >log 2>&1
+docker run -d -p 8822:22 --name "$name" "$name" >>log 2>&1
+ip_addr=$(hostname -i)
+ssh -o StrictHostKeyChecking=no -p 8822 root@"$ip_addr" hostname >>log 2>&1
+echo "ssh returned exit code $?"
+echo "=== cf-remote info ===" | tee -a log
+cf-remote --log-level DEBUG info -H root@"$ip_addr":8822 2>&1 | tee -a log
+echo "cf-remote info got return code $?"
+echo "=== cf-remote install ===" | tee -a log
+cf-remote --log-level DEBUG install --clients root@"$ip_addr":8822 2>&1 | tee -a log
+ssh -o StrictHostKeyChecking=no -p 8822 root@"$ip_addr" cf-agent -V >>log 2>&1

tests/docker/001-docker-install-test.sh

@@ -0,0 +1,10 @@
+FROM debian:12-slim
+
+# openssh-server depends on dbus and systemd which is a lot, so use dropbear instead
+RUN apt update -y && apt install -y dropbear openssh-sftp-server
+
+# setup no password login via ssh for root
+RUN sed -ie 's/^root:[^:]*:/root::/' /etc/shadow
+
+# run dropbear sshd in foreground (-F) and allow blank password logins (-B)
+CMD [ "dropbear", "-FB" ]

tests/docker/Dockerfile

@@ -0,0 +1,2 @@
+cf-remote install --clients localhost
+cf-agent -V