Download Sync server executable from artifacts

Instead of using Docker to run the Sync server, download its executable from GitLab artifacts and run it.

Author: Shubham

.gitlab-ci.yml

@@ -109,15 +109,15 @@ test-sync:linux:x64:
     matrix:
       - PYTHON_VERSION: [ '3.7', '3.8', '3.9', '3.10', '3.11', '3.12' ]
 
-test-sync:linux:armv7hf:
-  extends: .test-sync
-  tags: [ armv7hf, shell, linux, python3 ]
+#test-sync:linux:armv7hf:
+#  extends: .test-sync
+#  tags: [ armv7hf, shell, linux, python3 ]
 
 test-sync:linux:aarch64:
   extends: .test-sync
   tags: [ aarch64, shell, linux, python3 ]
 
-test-sync:mac:x64:
+test-sync:mac:arm64:
   extends: .test-sync
   script:
     - python3 -m venv .venv
@@ -126,11 +126,11 @@ test-sync:mac:x64:
     - rm -r objectbox  # todo this is ugly; let's copy required files in a sub-folder instead?
     - pip3 install --force-reinstall dist/*.whl  # Artifacts from build-sync
     - python -m pytest -s --runsync
-  tags: [ mac, x64, shell, python3 ]
+  tags: [ mac, arm64, shell, python3 ]
 
-test-sync:windows:x64:
-  extends: .test-sync
-  tags: [ windows, x64, python ]
-  variables:
-    PYTHON: "python.exe"
+#test-sync:windows:x64:
+#  extends: .test-sync
+#  tags: [ windows, x64, python ]
+#  variables:
+#    PYTHON: "python.exe"
 

tests/common.py

@@ -3,6 +3,12 @@
 import subprocess
 import socket
 import typing
+import platform
+import glob
+import zipfile
+import tarfile
+import signal
+import urllib.request
 
 import pytest
 import objectbox
@@ -48,56 +54,169 @@ def create_test_store(db_path: str = "testdata", clear_db: bool = True) -> objec
 
 @dataclass
 class SyncServerConfig:
-    container_id: str
+    pid: int
     port: int
 
 
+def _get_sync_server_job_name() -> str:
+    """Returns the GitLab CI job name for downloading the sync server based on OS and architecture."""
+    system = platform.system().lower()
+    machine = platform.machine().lower()
+
+    if system == "darwin":
+        if machine in ("arm64", "aarch64"):
+            return "b:mac-arm64-server"
+        else:
+            return "b:mac-x64-server"
+    elif system == "linux":
+        if machine in ("arm64", "aarch64"):
+            return "b:linux-aarch64-server"
+        else:
+            return "b:linux-x64-server"
+    else:
+        raise RuntimeError(f"Unsupported platform: {system} {machine}")
+
+
 def start_sync_server() -> typing.Union[SyncServerConfig, None]:
-    """ Starts the ObjectBox Sync Server in a Docker container. """
+    """ Downloads and starts the ObjectBox Sync Server binary. """
     current_dir = os.path.dirname(os.path.realpath(__file__))
-    user_id = None
-    if os.name != 'nt':
-        user_id = os.getuid()
-    else:
-        user_id = 0
+
+    # Check for required environment variables
+    ci_server_url = os.environ.get("CI_SERVER_URL")
+    ci_job_token = os.environ.get("CI_JOB_TOKEN")
+
+    if not ci_server_url or not ci_job_token:
+        test_logger.warning("CI_SERVER_URL or CI_JOB_TOKEN not set, cannot download sync server")
+        return None
+
     try:
-        command = ("docker run "
-                   "--rm "
-                   "-d "
-                   f"--volume {current_dir}:/data "
-                   f"--user {user_id} "
-                   "-p 127.0.0.1:9999:9999 "
-                   "objectboxio/sync-server-trial "
-                   "--conf sync_server_config.json")
-        logger.info("Using command to start Sync Server Docker container:" + command)
-        stdout = subprocess.run(command.split(), check=True, capture_output=True, text=True).stdout
-        container_id = stdout.strip()
+        # GitLab artifact download configuration
+        project_id = "4"
+        job_name = _get_sync_server_job_name()
+        branch = "syncdev"
+
+        # URL-encode the job name (colon -> %3A)
+        job_name_encoded = job_name.replace(":", "%3A")
+
+        artifact_url = f"{ci_server_url}/api/v4/projects/{project_id}/jobs/artifacts/{branch}/download?job={job_name_encoded}"
+
+        print(f"Downloading sync-server artifact from: {artifact_url}")
+
+        # Download the artifact
+        artifact_path = os.path.join(current_dir, "artifact.zip")
+        request = urllib.request.Request(artifact_url, headers={"JOB-TOKEN": ci_job_token})
+        with urllib.request.urlopen(request) as response, open(artifact_path, "wb") as out_file:
+            out_file.write(response.read())
+
+        # Extract the outer artifact zip
+        with zipfile.ZipFile(artifact_path, "r") as zip_ref:
+            zip_ref.extractall(current_dir)
+
+        # Find and extract the sync-server archive (could be .zip or .tar.gz)
+        artifacts_dir = os.path.join(current_dir, "artifacts")
+
+        # Look for both .zip and .tar.gz files
+        sync_server_zips = glob.glob(os.path.join(artifacts_dir, "objectbox-sync-server-*.zip"))
+        sync_server_tarballs = glob.glob(os.path.join(artifacts_dir, "objectbox-sync-server-*.tar.gz"))
+
+        if sync_server_zips:
+            sync_server_archive = sync_server_zips[0]
+            print(f"Found sync-server zip archive: {sync_server_archive}")
+            with zipfile.ZipFile(sync_server_archive, "r") as zip_ref:
+                zip_ref.extractall(current_dir)
+        elif sync_server_tarballs:
+            sync_server_archive = sync_server_tarballs[0]
+            print(f"Found sync-server tar.gz archive: {sync_server_archive}")
+            with tarfile.open(sync_server_archive, "r:gz") as tar_ref:
+                tar_ref.extractall(current_dir)
+        else:
+            raise RuntimeError("Could not find objectbox-sync-server-*.zip or *.tar.gz in artifacts")
+
+        # Verify sync-server executable exists
+        sync_server_executable = os.path.join(current_dir, "sync-server")
+        if platform.system().lower() == "windows":
+            sync_server_executable += ".exe"
+
+        if not os.path.exists(sync_server_executable):
+            raise RuntimeError("sync-server executable not found after extraction")
+
+        # Make executable on Unix systems
+        if os.name != "nt":
+            os.chmod(sync_server_executable, 0o755)
+
+        print("Starting sync-server in background...")
 
+        # Run sync-server in background
+        process = subprocess.Popen(
+            [
+                sync_server_executable,
+                "--model", os.path.join(current_dir, "objectbox-model.json"),
+                "--unsecured-no-authentication",
+                "--debug"
+            ],
+            cwd=current_dir,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL
+        )
+
+        print(f"Sync server started with PID: {process.pid}")
+
+        # Wait for server to be ready
+        time.sleep(2)
+
+        # Check if server is still running
+        if process.poll() is not None:
+            raise RuntimeError("Sync server failed to start")
+
+        # Wait for port to be available
         start_time = time.time()
         while (time.time() - start_time) < 10:
             try:
-                with socket.create_connection(("127.0.0.1", 9999)):
+                with socket.create_connection(("127.0.0.1", 9999), timeout=1):
                     break
-            except OSError:
-                pass
+            except (OSError, socket.timeout):
+                time.sleep(0.5)
         else:
             raise RuntimeError("Timed out waiting for Sync Server to start")
 
-        test_logger.info("Started ObjectBox Sync Server in Docker")
-        return SyncServerConfig(container_id=container_id, port=9999)
+        print("Sync server is running")
+        return SyncServerConfig(pid=process.pid, port=9999)
+
     except Exception as e:
-        test_logger.warning(f"Could not start ObjectBox Sync Server in Docker: {e}")
+        test_logger.warning(f"Could not start ObjectBox Sync Server: {e}")
         return None
 
 
-def stop_sync_server(container_id: str):
-    """ Stops the ObjectBox Sync Server Docker container. """
+def stop_sync_server(pid: int):
+    """ Stops the ObjectBox Sync Server process. """
     try:
-        command = f"docker stop {container_id}"
-        subprocess.run(command.split(), check=True)
-        test_logger.info("Stopped ObjectBox Sync Server Docker container")
+        print(f"Stopping sync server (PID: {pid})...")
+
+        # Send SIGTERM (or equivalent on Windows)
+        if os.name == "nt":
+            subprocess.run(["taskkill", "/F", "/PID", str(pid)], check=False)
+        else:
+            os.kill(pid, signal.SIGTERM)
+
+        # Wait for process to stop
+        for i in range(10):
+            try:
+                os.kill(pid, 0)  # Check if process exists
+                time.sleep(1)
+            except OSError:
+                print(f"Sync server stopped after {i + 1}s")
+                return
+
+        # Force kill if still running
+        print("Sync server still running after 10s, sending SIGKILL...")
+        if os.name == "nt":
+            subprocess.run(["taskkill", "/F", "/PID", str(pid)], check=False)
+        else:
+            os.kill(pid, signal.SIGKILL)
+
+        print("Stopped ObjectBox Sync Server")
     except Exception as e:
-        test_logger.warning(f"Could not stop ObjectBox Sync Server Docker container: {e}")
+        test_logger.warning(f"Could not stop ObjectBox Sync Server: {e}")
 
 
 def assert_equal_prop(actual, expected, default):

tests/conftest.py

@@ -92,7 +92,7 @@ def sync_server():
     server_config = start_sync_server()
     yield server_config
     if server_config:
-        stop_sync_server(server_config.container_id)
+        stop_sync_server(server_config.pid)
 
 
 def pytest_addoption(parser):