Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions .github/actions/run-test/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,15 @@ runs:
- name: Run Unittests
shell: bash
run: |
python -m unittest discover -v tests
PYTHONPATH=src python -m unittest discover -v tests

- name: Run Examples
shell: bash
run: |
uv pip install --system -r examples/applications/requirements_applications.txt
for example in "./examples"/*.py; do
echo "Running $example"
PYTHONPATH=. python $example
PYTHONPATH=src python $example
done
for example in "./examples/applications"/*.py; do
if python -c 'import sys; sys.exit(not sys.version_info <= (3, 10))'; then
Expand All @@ -42,5 +42,5 @@ runs:
fi

echo "Running $example"
PYTHONPATH=. python $example
PYTHONPATH=src python $example
done
6 changes: 4 additions & 2 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ if(NOT SKBUILD_PROJECT_NAME)
endif()

# Path to the version file
set(VERSION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/scaler/version.txt")
set(VERSION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/src/scaler/version.txt")

# Check if the version file exists
if(NOT EXISTS "${VERSION_FILE}")
Expand Down Expand Up @@ -118,8 +118,10 @@ message(STATUS "Python ABI: ${Python3_SOABI}")
# Make LSP happy
include_directories(${CAPNP_INCLUDE_DIRS})
include_directories(${PROJECT_SOURCE_DIR})
include_directories(${PROJECT_SOURCE_DIR}/src)
include_directories(${PROJECT_SOURCE_DIR}/src/cpp)

add_subdirectory(scaler)
add_subdirectory(src/cpp/scaler)

if(CMAKE_BUILD_TYPE STREQUAL "Debug")
add_subdirectory(tests)
Expand Down
4 changes: 2 additions & 2 deletions docs/source/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,15 @@
import os
import sys

sys.path.insert(0, os.path.abspath(os.path.join("..", "..")))
sys.path.insert(0, os.path.abspath(os.path.join("..", "..", "src")))


# -- Project information -----------------------------------------------------

project = "OpenGRIS Scaler"
author = "Citi"

with open("../../scaler/version.txt", "rt") as f:
with open("../../src/scaler/version.txt", "rt") as f:
version = f.read().strip()

release = f"{version}-py3-none-any"
Expand Down
10 changes: 8 additions & 2 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,13 @@ dev = [
[tool.scikit-build.metadata.version]
provider = "scikit_build_core.metadata.regex"
regex = '(?P<value>\d+\.\d+\.\d+(\.[1-9]\d*)?)'
input = "./scaler/version.txt"
input = "./src/scaler/version.txt"

[tool.setuptools]
package-dir = {"" = "src"}

[tool.setuptools.packages.find]
where = ["src"]

[project.urls]
Home = "https://github.com/Citi/scaler"
Expand All @@ -81,7 +87,7 @@ scaler_worker_adapter_symphony = "scaler.entry_points.worker_adapter_symphony:ma

[tool.scikit-build]
cmake.source-dir = "."
wheel.packages = ["scaler"]
wheel.packages = ["src/scaler"]
build.targets = ["py_ymq", "py_object_storage_server"]

[tool.scikit-build.cmake.define]
Expand Down
4 changes: 2 additions & 2 deletions scripts/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ BUILD_DIR="build_${OS}_${ARCH}"
BUILD_PRESET="${OS}-${ARCH}"

rm -rf $BUILD_DIR
rm -f scaler/protocol/capnp/*.c++
rm -f scaler/protocol/capnp/*.h
rm -f src/scaler/protocol/capnp/*.c++
rm -f src/scaler/protocol/capnp/*.h
echo "Build directory: $BUILD_DIR"
echo "Build preset: $BUILD_PRESET"

Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ add_library(ymq_objs OBJECT
simple_interface.cpp
)

set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/scaler/io/ymq)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/src/scaler/io/ymq)

if(LINUX)
# ymq python =======================================================================================================
Expand All @@ -70,7 +70,10 @@ if(LINUX)
pymod_ymq/ymq.cpp
)

# target_include_directories(py_ymq PRIVATE ${Python3_INCLUDE_DIRS})
target_include_directories(py_ymq PRIVATE
${PROJECT_SOURCE_DIR}/src/cpp
)

target_link_libraries(py_ymq PRIVATE
$<$<CXX_COMPILER_ID:GNU,Clang>:-Wl,-Bstatic> ymq_objs
$<$<CXX_COMPILER_ID:GNU,Clang>:-Wl,-Bdynamic>
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

// First-party
#include "scaler/io/ymq/error.h"
#include "scaler/io/ymq/typedefs.h"

struct YMQState {
OwnedPyObject<> enumModule; // Reference to the enum module
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
# capnp_objs ===========================================================================================================
# compile object storage schema capnp
set(CAPNPC_SRC_PREFIX "${PROJECT_SOURCE_DIR}/scaler/protocol/capnp" CACHE STRING "" FORCE)
set(CAPNPC_OUTPUT_DIR "${PROJECT_SOURCE_DIR}/scaler/protocol/capnp" CACHE STRING "" FORCE)
set(CAPNPC_SRC_PREFIX "${PROJECT_SOURCE_DIR}/src/scaler/protocol/capnp" CACHE STRING "" FORCE)
set(CAPNPC_OUTPUT_DIR "${PROJECT_SOURCE_DIR}/src/scaler/protocol/capnp" CACHE STRING "" FORCE)
file(MAKE_DIRECTORY "${CAPNPC_OUTPUT_DIR}")
capnp_generate_cpp(OBJECT_STORAGE_CAPNP_SRCS OBJECT_STORAGE_CAPNP_HEADERS "../protocol/capnp/object_storage.capnp")
capnp_generate_cpp(OBJECT_STORAGE_CAPNP_SRCS OBJECT_STORAGE_CAPNP_HEADERS "${PROJECT_SOURCE_DIR}/src/scaler/protocol/capnp/object_storage.capnp")

add_library(capnp_objs OBJECT
${OBJECT_STORAGE_CAPNP_SRCS}
Expand All @@ -18,7 +18,7 @@ add_library(object_storage_server_objs OBJECT
object_manager.cpp
)

set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/scaler/object_storage)
set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_SOURCE_DIR}/src/scaler/object_storage)

# object_storage_server python =========================================================================================
add_library(py_object_storage_server MODULE
Expand All @@ -30,6 +30,10 @@ set_target_properties(py_object_storage_server PROPERTIES
OUTPUT_NAME "object_storage_server"
)

target_include_directories(py_object_storage_server PRIVATE
${PROJECT_SOURCE_DIR}/src/cpp
)

target_link_libraries(py_object_storage_server PRIVATE
capnp_objs
ymq_objs
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#include "io_helper.h"
#include "scaler/object_storage/io_helper.h"

#include <netinet/in.h> // sockaddr_in
#include <sys/socket.h> // socket(2)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
#include <cstdio>
#include <vector>

#include "../protocol/capnp/object_storage.capnp.h"
#include "scaler/protocol/capnp/object_storage.capnp.h"

namespace scaler {
namespace object_storage {
Expand Down
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
3 changes: 0 additions & 3 deletions tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,4 @@ endfunction()
if(LINUX OR APPLE)
# Add the directory for the C++ tests.
add_subdirectory(cpp)

# Add the new directory for io tests.
add_subdirectory(io/ymq)
endif()
Empty file added tests/client/__init__.py
Empty file.
12 changes: 7 additions & 5 deletions tests/test_client.py → tests/client/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from scaler.utility.logging.scoped_logger import ScopedLogger
from scaler.utility.logging.utility import setup_logger
from scaler.worker.preload import PreloadSpecError, _parse_preload_spec, execute_preload
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name


def noop(sec: int):
Expand All @@ -34,7 +34,7 @@ def raise_exception(foo: int):

def get_preloaded_value():
"""Function that retrieves value set by preload"""
from tests.utility import get_global_value
from tests.utility.utility import get_global_value

return get_global_value()

Expand Down Expand Up @@ -364,7 +364,9 @@ def _create_preload_cluster(self, preload: str, logging_paths: tuple = ("/dev/st
return preload_cluster

def test_preload_success(self):
preload_cluster = self._create_preload_cluster(preload="tests.utility:setup_global_value('test_preload_value')")
preload_cluster = self._create_preload_cluster(
preload="tests.utility.utility:setup_global_value('test_preload_value')"
)

try:
preload_cluster.start()
Expand All @@ -390,7 +392,7 @@ def test_preload_failure(self):

try:
preload_cluster = self._create_preload_cluster(
preload="tests.utility:failing_preload()", logging_paths=(log_path,)
preload="tests.utility.utility:failing_preload()", logging_paths=(log_path,)
)

try:
Expand All @@ -406,7 +408,7 @@ def test_preload_failure(self):
processor_log_content += f.read()

# Verify that the preload failure was logged properly
self.assertIn("preloading: tests.utility:failing_preload with args", processor_log_content)
self.assertIn("preloading: tests.utility.utility:failing_preload with args", processor_log_content)

# If we reach here without any other exceptions, the test is successful
finally:
Expand Down
2 changes: 1 addition & 1 deletion tests/test_future.py → tests/client/test_future.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
from scaler.utility.identifiers import ClientID, ObjectID, TaskID
from scaler.utility.logging.utility import setup_logger
from scaler.utility.network_util import get_available_tcp_port
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name


def noop_sleep(sec: int):
Expand Down
Empty file added tests/cluster/__init__.py
Empty file.
2 changes: 1 addition & 1 deletion tests/test_cluster.py → tests/cluster/test_cluster.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
from scaler import Client, SchedulerClusterCombo
from scaler.utility.logging.utility import setup_logger
from scaler.utility.network_util import get_available_tcp_port
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name


class TestCluster(unittest.TestCase):
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from scaler import Client, Cluster, SchedulerClusterCombo
from scaler.config.defaults import DEFAULT_LOGGING_PATHS
from scaler.utility.logging.utility import setup_logger
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name


def noop_sleep(sec: int):
Expand Down
Empty file added tests/config/__init__.py
Empty file.
File renamed without changes.
Empty file added tests/core/__init__.py
Empty file.
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

from scaler.utility.logging.utility import setup_logger
from scaler.utility.queues.async_indexed_queue import AsyncIndexedQueue
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name


class TestAsyncIndexedQueue(unittest.TestCase):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

from scaler.utility.logging.utility import setup_logger
from scaler.utility.queues.async_priority_queue import AsyncPriorityQueue
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name


class TestAsyncPriorityQueue(unittest.TestCase):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

from scaler.utility.logging.utility import setup_logger
from scaler.utility.queues.async_sorted_priority_queue import AsyncSortedPriorityQueue
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name


class TestSortedPriorityQueue(unittest.TestCase):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from scaler.config.types.zmq import ZMQConfig
from scaler.utility.logging.utility import setup_logger
from scaler.utility.network_util import get_available_tcp_port
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name

# This is a manual test because it can loop infinitely if it fails

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from scaler.utility.logging.utility import setup_logger
from scaler.utility.queues.indexed_queue import IndexedQueue
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name


class TestIndexedQueue(unittest.TestCase):
Expand Down
2 changes: 1 addition & 1 deletion tests/test_profiling.py → tests/core/test_profiling.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from scaler import Client, SchedulerClusterCombo
from scaler.utility.logging.utility import setup_logger
from scaler.utility.network_util import get_available_tcp_port
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name


def dummy(n: int):
Expand Down
2 changes: 1 addition & 1 deletion tests/test_protected.py → tests/core/test_protected.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from scaler import Client, SchedulerClusterCombo
from scaler.utility.logging.utility import setup_logger
from scaler.utility.network_util import get_available_tcp_port
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name


class TestProtected(unittest.TestCase):
Expand Down
File renamed without changes.
Empty file added tests/graph/__init__.py
Empty file.
2 changes: 1 addition & 1 deletion tests/test_graph.py → tests/graph/test_graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from scaler.utility.graph.optimization import cull_graph
from scaler.utility.logging.scoped_logger import ScopedLogger
from scaler.utility.logging.utility import setup_logger
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name


def inc(i):
Expand Down
1 change: 0 additions & 1 deletion tests/io/ymq/CMakeLists.txt

This file was deleted.

Empty file.
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

from scaler.scheduler.object_usage.object_tracker import ObjectTracker, ObjectUsage
from scaler.utility.logging.utility import setup_logger
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name


@dataclasses.dataclass
Expand Down
Empty file added tests/scheduler/__init__.py
Empty file.
2 changes: 1 addition & 1 deletion tests/test_balance.py → tests/scheduler/test_balance.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from scaler.config.defaults import DEFAULT_LOAD_BALANCE_SECONDS
from scaler.utility.logging.utility import setup_logger
from scaler.utility.network_util import get_available_tcp_port
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name


def sleep_and_return_pid(sec: int):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from scaler import Client, Cluster, SchedulerClusterCombo
from scaler.scheduler.allocate_policy.allocate_policy import AllocatePolicy
from scaler.utility.logging.utility import setup_logger
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name


class TestCapabilities(unittest.TestCase):
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from scaler.scheduler.allocate_policy.capability_allocate_policy import CapabilityAllocatePolicy
from scaler.utility.identifiers import ClientID, TaskID, WorkerID
from scaler.utility.logging.utility import setup_logger
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name

MAX_TASKS_PER_WORKER = 10

Expand Down
2 changes: 1 addition & 1 deletion tests/test_scaling.py → tests/scheduler/test_scaling.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
from scaler.utility.logging.utility import setup_logger
from scaler.utility.network_util import get_available_tcp_port
from scaler.worker_adapter.native import NativeWorkerAdapter
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name


def _run_native_worker_adapter(address: str, webhook_port: int) -> None:
Expand Down
Empty file added tests/serializer/__init__.py
Empty file.
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from scaler.utility.logging.scoped_logger import ScopedLogger
from scaler.utility.logging.utility import setup_logger
from scaler.utility.network_util import get_available_tcp_port
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name


def noop(sec: int):
Expand Down
Empty file added tests/task/__init__.py
Empty file.
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from scaler import Client, SchedulerClusterCombo
from scaler.utility.logging.utility import setup_logger
from tests.utility import logging_test_name
from tests.utility.utility import logging_test_name

N_TASKS = 30
N_WORKERS = 3
Expand Down
Empty file added tests/ui/__init__.py
Empty file.
Loading
Loading