Merge pull request #231 from KatharaFramework/develop

Kathará v3.6.3

docs/kathara-exec.1.ronn

@@ -5,7 +5,7 @@ kathara-exec(1) -- Execute a command in a Kathara device
 ## SYNOPSIS
 
 `kathara exec` [`-h`] [`-d` <DIRECTORY> \| `-v`]  
-[`--no-stdout`] [`--no-stderr`]  
+[`--no-stdout`] [`--no-stderr`] [`--wait`]  
 <DEVICE_NAME> <COMMAND> [<COMMAND> ...]
 
 ## DESCRIPTION
@@ -35,6 +35,11 @@ Execute a command in the Kathara device DEVICE_NAME.
 * `--no-stderr`:
     Disable stderr of the executed command.
 
+* `--wait`:
+    Wait until startup commands execution finishes.
+
+    You can override the wait by pressing `[ENTER]`.
+
 * `<DEVICE_NAME>:
     Name of the device to execute the command into.
 

docs/kathara-lab.conf.5.ronn

@@ -27,12 +27,13 @@ In order to establish a uniform convention, comment lines should always start wi
 * `cpus` (float):
     Limit the amount of CPU available for this device.
 
-    This option takes a positive float, ranging from 0 to max number of host logical CPUs. For instance, if the host device has two CPUs and you set `device[cpus]=1.5`, the container is guaranteed at most one and a half of the CPUs.
+    This option takes a positive float, ranging from 0 to max number of host logical CPUs. For instance, if the host device has two CPUs and you set `device[cpus]=1.5`, the device is guaranteed at most one and a half of the CPUs.
 
 * `port` (string):
-    Map localhost port HOST to the internal port GUEST of the device for the specified PROTOCOL.
+    Map localhost port HOST to the internal port GUEST of the device for the specified PROTOCOL. The syntax is [HOST:]GUEST[/PROTOCOL].
 
     If HOST port is not specified, default is 3000. If PROTOCOL is not specified, default is `tcp`. Supported PROTOCOL values are: tcp, udp, or sctp.
+    For instance, with this command you can map host's port 8080 to device's port 80 with TCP protocol: `device[port]="8080:80/tcp"`.
 
 * `bridged` (boolean):
     Connect the device to the host network by adding an additional network interface. This interface will be connected to the host network through a NAT connection.

docs/kathara-vstart.1.ronn

@@ -94,9 +94,10 @@ Notice: unless differently stated, command line arguments (DEVICE_NAME) and opti
     Connect the device to the host network by adding an additional network interface (will be the last one). This interface will be connected to the host network through a NAT connection and will receive its IP configuration automatically via DHCP.
 
 * `--port` <[HOST\:]GUEST[/PROTOCOL]> [<[HOST\:]GUEST[/PROTOCOL]> ...]:
-    Map localhost port HOST to the internal port GUEST of the device for the specified PROTOCOL.
+    Map localhost port HOST to the internal port GUEST of the device for the specified PROTOCOL. The syntax is [HOST:]GUEST[/PROTOCOL].
 
     If HOST port is not specified, default is 3000. If PROTOCOL is not specified, default is `tcp`. Supported PROTOCOL values are: tcp, udp, or sctp.
+    For instance, with this command you can map host's port 8080 to device's port 80 with TCP protocol: `--port "8080:80/tcp"`.
 
 * `--sysctl` <SYSCTL> [<SYSCTL> ...]:
     Set a sysctl option for the device. Only the `net.` namespace is allowed to be set.

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "kathara"
-version = "3.6.2"
+version = "3.6.3"
 description = "A lightweight container-based emulation system."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -40,6 +40,7 @@ dependencies = [
     "pyroute2>=0.5.19",
     "progressbar2>=1.14.0",
     "fs>=2.4.16",
+    "chardet",
     "libtmux>=0.8.2; platform_system == 'darwin' or platform_system == 'linux'",
     "appscript>=1.1.0; platform_system == 'darwin'",
     "pypiwin32>=223; platform_system == 'win32'",

scripts/Linux-Deb/Makefile

@@ -1,13 +1,13 @@
 #!/usr/bin/make -f
 
-VERSION=3.6.2
+VERSION=3.6.3
 DEBIAN_PACKAGE_VERSION=1
 LAUNCHPAD_NAME=user
 NO_BINARY_PACKAGES=pyroute2|pyuv|deepdiff
 
 .PHONY: allSigned docker-build-image prepare-deb-source unpack-deb-source clean-output-folder copy-debian-folder download-pip build-man build-deb-unsigned build-deb-signed ppa clean venv autocompletion
 
-allSigned: clean prepare-deb-source docker-signed_focal docker-signed_jammy docker-signed_kinetic
+allSigned: clean prepare-deb-source docker-signed_focal docker-signed_jammy
 
 docker-unsigned_%: clean prepare-deb-source docker-build-image_%
 	docker run -ti --rm -v `pwd`/../../:/opt/kathara kathara/linux-build-deb:$* /bin/bash -c \

scripts/Linux-Deb/debian/changelog

@@ -1,8 +1,12 @@
 kathara (__VERSION__-__DEBIAN_PACKAGE_VERSION____UBUNTU_VERSION__) __UBUNTU_VERSION__; urgency=low
 
-    * Fix FilesystemMixin APIs for file modifications
-    * Add a warning if a meta is repeated on the same device
-    * Add support for Docker Images tags in "kathara settings"
+    * Use "chardet" to parse all byte strings in order to correctly encode non-UTF8 characters
+    * Wait for startup commands execution while opening a connection to a device (the wait can be overridden by pressing [ENTER])
+    * Keep correct folders/files permissions when copying files inside the device
+    * Fix "sysctl" metadata parsing for negative values
+    * Fix "machine.is_ipv6_enabled()"" method to correctly follow priority of "ipv6" meta
+    * Rename "startup_commands" meta to "exec_commands" to avoid ambiguity with ".startup" files
+    * Add new helper methods to "model.Lab"
     * Minor fixes
 
  -- Kathara Team <[email protected]>  __DATE__

scripts/Linux-Pkg/Makefile

@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 
-VERSION=3.6.2
+VERSION=3.6.3
 PACKAGE_VERSION=1
 AUR_NAME=user
 AUR_MAIL[email protected]

scripts/Linux-Pkg/pkginfo/kathara.changelog

@@ -1,7 +1,11 @@
 __DATE__ Kathara Team <******@kathara.org>
 
 * Release v__VERSION__
-    * Fix FilesystemMixin APIs for file modifications
-    * Add a warning if a meta is repeated on the same device
-    * Add support for Docker Images tags in "kathara settings"
+    * Use "chardet" to parse all byte strings in order to correctly encode non-UTF8 characters
+    * Wait for startup commands execution while opening a connection to a device (the wait can be overridden by pressing [ENTER])
+    * Keep correct folders/files permissions when copying files inside the device
+    * Fix "sysctl" metadata parsing for negative values
+    * Fix "machine.is_ipv6_enabled()"" method to correctly follow priority of "ipv6" meta
+    * Rename "startup_commands" meta to "exec_commands" to avoid ambiguity with ".startup" files
+    * Add new helper methods to "model.Lab"
     * Minor fixes

scripts/Linux-Rpm/Makefile

@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 
-VERSION=3.6.2
+VERSION=3.6.3
 PACKAGE_VERSION=1
 
 .PHONY: all clean docker-build-image prepare-source prepare-man-pages prepare-bash-completion pack-source build

scripts/Linux-Rpm/rpm/kathara.spec

@@ -68,7 +68,11 @@ chmod g+s %{_libdir}/kathara/kathara
 
 %changelog
 *  __DATE__ Kathara Team <******@kathara.org> - __VERSION__-__PACKAGE_VERSION__
-- Fix FilesystemMixin APIs for file modifications
-- Add a warning if a meta is repeated on the same device
-- Add support for Docker Images tags in "kathara settings"
+- Use "chardet" to parse all byte strings in order to correctly encode non-UTF8 characters
+- Wait for startup commands execution while opening a connection to a device (the wait can be overridden by pressing [ENTER])
+- Keep correct folders/files permissions when copying files inside the device
+- Fix "sysctl" metadata parsing for negative values
+- Fix "machine.is_ipv6_enabled()"" method to correctly follow priority of "ipv6" meta
+- Rename "startup_commands" meta to "exec_commands" to avoid ambiguity with ".startup" files
+- Add new helper methods to "model.Lab"
 - Minor fixes

scripts/OSX/Makefile

@@ -1,7 +1,7 @@
 #!/usr/bin/make -s
 
 PRODUCT=Kathara
-VERSION=3.6.2
+VERSION=3.6.3
 TARGET_DIRECTORY=Output
 APPLE_DEVELOPER_CERTIFICATE_ID=FakeID
 ROFF_DIR=../../docs/Roff

scripts/Windows/WindowsBuild.bat

@@ -1,7 +1,11 @@
 set VENV_DIR=%cd%\venv
 
+rmdir /S /Q %VENV_DIR%
+
 python3.10 -m venv %VENV_DIR%
+if %errorlevel% neq 0 exit /b %errorlevel%
 CALL %VENV_DIR%\Scripts\activate
+if %errorlevel% neq 0 exit /b %errorlevel%
 
 pip install win_inet_pton
 pip install pyinstaller

scripts/Windows/installer.iss

@@ -2,7 +2,7 @@
 ; SEE THE DOCUMENTATION FOR DETAILS ON CREATING INNO SETUP SCRIPT FILES!
 
 #define MyAppName "Kathara"
-#define MyAppVersion "3.6.2"
+#define MyAppVersion "3.6.3"
 #define MyAppPublisher "Kathara Team"
 #define MyAppURL "https://www.kathara.org"
 #define MyAppExeName "kathara.exe"

scripts/pydoc/Makefile

@@ -2,23 +2,15 @@
 
 .PHONY: all doc verify-doc clean
 
-all: clean doc verify-doc
+all: doc
 
 doc: clean verify-doc
 	python3 generate_doc.py
 	cd docs && rm Kathara.model.md Kathara.setting.md Kathara.manager.md
-	sed '/manager.md/d' ./docs/Kathara-API-Docs.md
-	sed '/setting.md/d' ./docs/Kathara-API-Docs.md
-	sed '/docker.md/d' ./docs/Kathara-API-Docs.md
-	sed '/kubernetes.md/d' ./docs/Kathara-API-Docs.md
-	sed '/KubernetesSettingsAddon.md/d' ./docs/Kathara-API-Docs.md
-	sed '/DockerSettingsAddon.md/d' ./docs/Kathara-API-Docs.md
-	sed '/model.md/d' ./docs/Kathara-API-Docs.md
-	sed 's/\.\///g' ./docs/Kathara-API-Docs.md
-	sed 's/\.md//g' ./docs/Kathara-API-Docs.md
-	sed '/Modules/,/Classes/d' ./docs/Kathara-API-Docs.md
-	sed '/Functions/q' ./docs/Kathara-API-Docs.md
-	sed '/Functions/d' ./docs/Kathara-API-Docs.md
+	sed -i 's/\.\///' docs/Kathara-API-Docs.md
+	sed -i 's/\.md//' docs/Kathara-API-Docs.md
+	sed -i '/Modules/,/Classes/{/Classes/!d;}' docs/Kathara-API-Docs.md
+	rm docs/.pages
 
 verify-doc:
 	python3 -m pip install lazydocs pydocstyle

scripts/pydoc/generate_doc.py

@@ -1,12 +1,16 @@
 from lazydocs import generate_docs
 
 ignored_modules = [
-    "Kathara.version", "Kathara.auth", "Kathara.cli", "Kathara.event", "Kathara.foundation",
-    "Kathara.manager.docker.terminal", "Kathara.manager.kubernetes.terminal", "Kathara.os",
-    "Kathara.parser", "Kathara.test", "Kathara.trdparty", "Kathara.validator", "Kathara.kathara", "Kathara.decorators",
-    "Kathara.exceptions", "Kathara.strings", "Kathara.utils", "Kathara.webhooks", "kathara"
+    "Kathara.version", "Kathara.auth", "Kathara.cli", "Kathara.trdparty", "Kathara.foundation.cli",
+    "Kathara.foundation.test", "Kathara.exceptions", "Kathara.webhooks", "Kathara.validator", "Kathara.test", "kathara",
+    "Kathara.manager.kubernetes.terminal.KubernetesWSTerminal", "Kathara.foundation.factory",
+    "Kathara.manager.docker.terminal.DockerTTYTerminal",
+    "Kathara.foundation.manager.terminal.Terminal", "Kathara.foundation.manager.ManagerFactory",
+    "Kathara.foundation.setting.SettingsAddon", "Kathara.foundation.setting.SettingsAddonFactory",
+    "Kathara.setting.addon.DockerSettingsAddon", "Kathara.setting.addon.KubernetesSettingsAddon"
 ]
 
-generate_docs(["../../src"], src_base_url="https://github.com/KatharaFramework/Kathara/tree/master", output_path="./docs",
+generate_docs(["../../src"],
+              src_base_url="https://github.com/KatharaFramework/Kathara/tree/main", output_path="./docs",
               ignored_modules=ignored_modules,
               overview_file="Kathara-API-Docs.md", remove_package_prefix=False)

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = kathara
-version = 3.6.2
+version = 3.6.3
 author = Kathara Framework
 author_email = [email protected]
 description = A lightweight container based emulation system

setup.py

@@ -7,13 +7,13 @@
     package_dir={'': 'src'},
     packages=find_packages('src'),
     py_modules=['kathara'],
-    version='3.6.2',
+    version='3.6.3',
     license='gpl-3.0',
     description='A lightweight container based emulation system.',
     author='Kathara Framework',
     author_email='[email protected]',
     url='https://www.kathara.org',
-    download_url='https://github.com/KatharaFramework/Kathara/archive/refs/tags/3.6.2.tar.gz',
+    download_url='https://github.com/KatharaFramework/Kathara/archive/refs/tags/3.6.3.tar.gz',
     keywords=['NETWORK-EMULATION', 'CONTAINERS', 'NFV'],
     install_requires=[
         "binaryornot>=0.4.4",
@@ -27,6 +27,7 @@
         "pyroute2>=0.5.19",
         "progressbar2>=1.14.0",
         "fs>=2.4.16",
+        "chardet",
         "libtmux>=0.8.2; platform_system == 'darwin' or platform_system == 'linux'",
         "appscript>=1.1.0; platform_system == 'darwin'",
         "pypiwin32>=223; platform_system == 'win32'",

src/Kathara/cli/command/ExecCommand.py

@@ -2,6 +2,8 @@
 import sys
 from typing import List
 
+import chardet
+
 from ... import utils
 from ...foundation.cli.command.Command import Command
 from ...manager.Kathara import Kathara
@@ -52,6 +54,13 @@ def __init__(self) -> None:
             action="store_true",
             help='Disable stderr of the executed command.',
         )
+        self.parser.add_argument(
+            '--wait',
+            dest="wait",
+            action="store_true",
+            default=False,
+            help='Wait until startup commands execution finishes.',
+        )
         self.parser.add_argument(
             'machine_name',
             metavar='DEVICE_NAME',
@@ -78,17 +87,24 @@ def run(self, current_path: str, argv: List[str]) -> None:
             except (Exception, IOError):
                 lab = Lab(None, path=lab_path)
 
-        exec_output = Kathara.get_instance().exec(args['machine_name'], args['command'], lab_hash=lab.hash)
+        exec_output = Kathara.get_instance().exec(
+            args['machine_name'],
+            args['command'] if len(args['command']) > 1 else args['command'].pop(),
+            lab_hash=lab.hash,
+            wait=args['wait']
+        )
 
         try:
             while True:
                 (stdout, stderr) = next(exec_output)
-                stdout = stdout.decode('utf-8') if stdout else ""
-                stderr = stderr.decode('utf-8') if stderr else ""
 
                 if not args['no_stdout']:
+                    stdout_char_encoding = chardet.detect(stdout) if stdout else None
+                    stdout = stdout.decode(stdout_char_encoding['encoding']) if stdout else ""
                     sys.stdout.write(stdout)
                 if stderr and not args['no_stderr']:
+                    stderr_char_encoding = chardet.detect(stderr) if stderr else None
+                    stderr = stderr.decode(stderr_char_encoding['encoding']) if stderr else ""
                     sys.stderr.write(stderr)
         except StopIteration:
             pass

src/Kathara/cli/command/WipeCommand.py

@@ -1,5 +1,4 @@
 import argparse
-import shutil
 import sys
 from typing import List
 
@@ -67,6 +66,3 @@ def run(self, current_path: str, argv: List[str]) -> None:
                 raise PrivilegeError("You must be root in order to wipe all Kathara devices of all users.")
 
             Kathara.get_instance().wipe(all_users=bool(args['all']))
-
-            vlab_dir = utils.get_vlab_temp_path(force_creation=False)
-            shutil.rmtree(vlab_dir, ignore_errors=True)

src/Kathara/cli/ui/event/OpenMachineTerminal.py → src/Kathara/cli/ui/event/HandleMachineTerminal.py

@@ -1,9 +1,11 @@
+import sys
+
 from ..utils import open_machine_terminal
 from ....model import Machine as MachinePackage
 from ....setting.Setting import Setting
 
 
-class OpenMachineTerminal(object):
+class HandleMachineTerminal(object):
     """Listener fired when a device is deployed and started."""
 
     def run(self, item: 'MachinePackage.Machine') -> None:
@@ -18,3 +20,22 @@ def run(self, item: 'MachinePackage.Machine') -> None:
         if Setting.get_instance().open_terminals:
             for i in range(0, item.get_num_terms()):
                 open_machine_terminal(item)
+
+    def flush(self) -> None:
+        """Clean the stdout buffer.
+
+        Returns:
+            None
+        """
+        sys.stdout.write("\033[2J")
+        sys.stdout.write("\033[0;0H")
+        sys.stdout.flush()
+
+    def print_wait_msg(self) -> None:
+        """Print the startup commands waiting message.
+
+        Returns:
+            None
+        """
+        sys.stdout.write("Waiting startup commands execution. Press [ENTER] to override...")
+        sys.stdout.flush()

src/Kathara/cli/ui/event/register.py

@@ -1,6 +1,6 @@
-from .UpdateDockerImage import UpdateDockerImage
-from .OpenMachineTerminal import OpenMachineTerminal
+from .HandleMachineTerminal import HandleMachineTerminal
 from .HandleProgressBar import HandleProgressBar
+from .UpdateDockerImage import UpdateDockerImage
 from ....event.EventDispatcher import EventDispatcher
 
 
@@ -39,4 +39,7 @@ def _register_machine_events() -> None:
     EventDispatcher.get_instance().register("machine_undeployed", machine_undeploy_progress_bar_handler, "update")
     EventDispatcher.get_instance().register("machines_undeploy_ended", machine_undeploy_progress_bar_handler, "finish")
 
-    EventDispatcher.get_instance().register("machine_deployed", OpenMachineTerminal())
+    machine_terminal_handler = HandleMachineTerminal()
+    EventDispatcher.get_instance().register("machine_deployed", machine_terminal_handler, "run")
+    EventDispatcher.get_instance().register("machine_startup_wait_started", machine_terminal_handler, "print_wait_msg")
+    EventDispatcher.get_instance().register("machine_startup_wait_ended", machine_terminal_handler, "flush")