From 79098f40413217b4eb881240ec913c3ae883f182 Mon Sep 17 00:00:00 2001 From: I-Need-Sleep-Asap Date: Thu, 9 Oct 2025 21:02:43 +0530 Subject: [PATCH] Auto update toml file by adding a script for sphinx docs --- .gitignore | 2 + README.md | 10 +- docs/source/_scripts/generate_config.py | 162 ++++++++++++++++++ docs/source/conf.py | 9 + docs/source/tutorials/configuration.rst | 30 +--- scaler/config/mixins.py | 16 ++ scaler/config/section/cluster.py | 2 + .../config/section/native_worker_adapter.py | 2 + .../config/section/object_storage_server.py | 2 + scaler/config/section/scheduler.py | 2 + .../config/section/symphony_worker_adapter.py | 2 + scaler/config/section/top.py | 2 + scaler/config/section/webui.py | 2 + 13 files changed, 215 insertions(+), 28 deletions(-) create mode 100644 docs/source/_scripts/generate_config.py diff --git a/.gitignore b/.gitignore index 31a18b32b..c99993855 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,5 @@ CMakeFiles/ .vs/ scaler/protocol/capnp/*.c++ scaler/protocol/capnp/*.h + +docs/source/_static/example_config.toml diff --git a/README.md b/README.md index fe5662dda..448668132 100644 --- a/README.md +++ b/README.md @@ -247,7 +247,15 @@ The following table maps each Scaler command to its corresponding section name i #### Scenario 1: Unified Configuration File -Here is an example of a single `example_config.toml` file that configures multiple components using sections. +To get a complete template with all available options and their default values, you can generate a `example_config.toml` file by running the script included in the `docs` folder: + +```bash +python3 docs/source/_scripts/generate_config.py +``` + +This will create a `example_config.toml` file, which you can then customize. + +For simplicity, here is a condensed example of a `example_config.toml` file that configures multiple components: **example_config.toml** diff --git a/docs/source/_scripts/generate_config.py b/docs/source/_scripts/generate_config.py new file mode 100644 index 000000000..bfa4cb766 --- /dev/null +++ b/docs/source/_scripts/generate_config.py @@ -0,0 +1,162 @@ +import dataclasses +import enum +import importlib +import inspect +import pathlib +import pkgutil +import sys +from typing import Any, Dict, Tuple, Type, Union, get_args, get_origin + +import scaler.config.section +from scaler.config.mixins import ConfigType + + +def find_project_root(marker: str = "pyproject.toml") -> pathlib.Path: + """Searches upwards from the current script for a project root marker.""" + current_dir = pathlib.Path(__file__).parent.resolve() + while current_dir != current_dir.parent: + if (current_dir / marker).exists(): + return current_dir + current_dir = current_dir.parent + raise FileNotFoundError(f"Could not find the project root marker '{marker}'") + + +try: + SCRIPT_DIR = pathlib.Path(__file__).parent.resolve() + PROJECT_ROOT = find_project_root() + SCALER_ROOT = PROJECT_ROOT / "scaler" +except FileNotFoundError as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + + +def get_config_classes() -> Dict[str, Type]: + """Dynamically finds all configuration section classes using pkgutil.""" + if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + + config_classes = {} + + for module_info in pkgutil.iter_modules(scaler.config.section.__path__, f"{scaler.config.section.__name__}."): + try: + module = importlib.import_module(module_info.name) + for name, obj in inspect.getmembers(module, inspect.isclass): + + if obj.__module__ == module_info.name and hasattr(obj, "_is_config_section") and obj._is_config_section: + # We still check if it's a dataclass, because the rest of + # the script depends on dataclasses.fields() + if not dataclasses.is_dataclass(obj): + print( + f"Warning: Config section {obj} is marked with @config_section " + "but is not a dataclass. Skipping." + ) + continue + + section_name = module_info.name.split(".")[-1] + config_classes[section_name] = obj + + except ImportError as e: + print(f"Warning: Could not import module {module_info.name}. Skipping. Error: {e}") + return config_classes + + +def get_type_repr(type_hint: Any) -> str: + """Creates a user-friendly string representation of a type hint.""" + if inspect.isclass(type_hint): + if issubclass(type_hint, ConfigType): + return "str" + if issubclass(type_hint, enum.Enum): + choices = ", ".join(e.name for e in type_hint) + return f"str (choices: {choices})" + + origin = get_origin(type_hint) + args = get_args(type_hint) + + if origin is Union and len(args) == 2 and args[1] is type(None): + return f"Optional[{get_type_repr(args[0])}]" + + if origin in (tuple, Tuple) and len(args) == 2 and args[1] is Ellipsis: + return f"Tuple[{get_type_repr(args[0])}, ...]" + + if origin: + origin_name = getattr(origin, "__name__", str(origin)).capitalize() + args_repr = ", ".join(get_type_repr(arg) for arg in args) + return f"{origin_name}[{args_repr}]" + + return getattr(type_hint, "__name__", str(type_hint)) + + +def format_value(value: Any) -> str: + """Formats Python values into valid TOML strings.""" + if isinstance(value, (ConfigType, str)): + return f'"{value}"' + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, enum.Enum): + return f'"{value.name}"' + if value is None: + return '""' + if isinstance(value, (list, tuple)): + return f"[{', '.join(format_value(v) for v in value)}]" + if dataclasses.is_dataclass(value): + fields = [f"{f.name} = {format_value(getattr(value, f.name))}" for f in dataclasses.fields(value)] + return f"{{ {', '.join(fields)} }}" + return str(value) + + +def generate_toml_string(config_classes: Dict[str, Type]) -> str: + """Generates the full TOML configuration string.""" + toml_lines = [ + "# Default configuration for OpenGRIS Scaler", + "# This file is auto-generated. Do not edit manually.", + "# For required fields, a value must be provided.", + "", + ] + + for section_name, config_class in sorted(config_classes.items()): + toml_lines.append(f"[{section_name}]") + for field in dataclasses.fields(config_class): + type_repr = get_type_repr(field.type) + + if field.default is dataclasses.MISSING and field.default_factory is dataclasses.MISSING: + toml_lines.append(f"# Type: {type_repr} (REQUIRED)") + toml_lines.append(f'{field.name} = ""') + continue + + toml_lines.append(f"# Type: {type_repr}") + + default_value = field.default + if field.default_factory is not dataclasses.MISSING: + try: + default_value = field.default_factory() + except Exception as e: + print(f"Warning: Could not execute default_factory for {field.name}. Error: {e}") + continue + + formatted_default = format_value(default_value) + toml_lines.append(f"{field.name} = {formatted_default}") + toml_lines.append("") + + return "\n".join(toml_lines) + + +def main(): + classes = get_config_classes() + if not classes: + print("Error: No configuration classes found.", file=sys.stderr) + sys.exit(1) + + toml_content = generate_toml_string(classes) + + output_dir = SCRIPT_DIR.parent / "_static" + output_dir.mkdir(exist_ok=True) + output_path = output_dir / "example_config.toml" + + with open(output_path, "w") as f: + f.write(toml_content) + + print(f"Successfully generated docs TOML config at: {output_path}") + + +if __name__ == "__main__": + main() diff --git a/docs/source/conf.py b/docs/source/conf.py index ea1a12b00..8a3623123 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -13,6 +13,7 @@ import os import sys + sys.path.insert(0, os.path.abspath(os.path.join("..", ".."))) @@ -31,6 +32,14 @@ .. |release| replace:: {release} """ +# -- Auto-generate TOML config for docs -------------------------------------- +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "_scripts"))) + +import generate_config # noqa: E402 +print("Executing script to generate TOML config...") +generate_config.main() +# --------------------------------------------------------------------------- + # -- General configuration --------------------------------------------------- # Add any Sphinx extension module names here, as strings. They can be diff --git a/docs/source/tutorials/configuration.rst b/docs/source/tutorials/configuration.rst index ed13311bc..62ebe55ac 100644 --- a/docs/source/tutorials/configuration.rst +++ b/docs/source/tutorials/configuration.rst @@ -190,34 +190,10 @@ Practical Scenarios & Examples **Scenario 1: Unified Configuration File** -Here is an example of a single ``example_config.toml`` file that configures multiple components using sections. +Here is an example of a single ``example_config.toml`` file with their default values. This file is automatically generated from the source code. -**example_config.toml** - -.. code-block:: toml - - # This is a unified configuration file for all Scaler components. - - [scheduler] - scheduler_address = "tcp://127.0.0.1:6378" - object_storage_address = "tcp://127.0.0.1:6379" - monitor_address = "tcp://127.0.0.1:6380" - allocate_policy = "even" - logging_level = "INFO" - logging_paths = ["/dev/stdout", "/var/log/scaler/scheduler.log"] - - [cluster] - scheduler_address = "tcp://127.0.0.1:6378" - num_of_workers = 8 - per_worker_capabilities = "linux,cpu=8" - task_timeout_seconds = 600 - - [object_storage_server] - object_storage_address = "tcp://127.0.0.1:6379" - - [webui] - monitor_address = "tcp://127.0.0.1:6380" - web_port = 8081 +.. literalinclude:: /_static/example_config.toml + :language: toml With this single file, starting your entire stack is simple and consistent: diff --git a/scaler/config/mixins.py b/scaler/config/mixins.py index 5628bd5b1..0b9f51244 100644 --- a/scaler/config/mixins.py +++ b/scaler/config/mixins.py @@ -18,3 +18,19 @@ def from_string(cls, value: str) -> Self: @abc.abstractmethod def __str__(self) -> str: pass + + +def config_section(cls): + """ + A class decorator to explicitly mark a dataclass + as a configuration section. It is used for sphinx + docs generation. + + Usage: + @config_section + @dataclass + class MyConfig: + ... + """ + setattr(cls, "_is_config_section", True) + return cls diff --git a/scaler/config/section/cluster.py b/scaler/config/section/cluster.py index bbcf3708d..9cd5d6c6d 100644 --- a/scaler/config/section/cluster.py +++ b/scaler/config/section/cluster.py @@ -2,12 +2,14 @@ from typing import Optional, Tuple from scaler.config import defaults +from scaler.config.mixins import config_section from scaler.config.types.object_storage_server import ObjectStorageConfig from scaler.config.types.worker import WorkerCapabilities, WorkerNames from scaler.config.types.zmq import ZMQConfig from scaler.utility.logging.utility import LoggingLevel +@config_section @dataclasses.dataclass class ClusterConfig: scheduler_address: ZMQConfig diff --git a/scaler/config/section/native_worker_adapter.py b/scaler/config/section/native_worker_adapter.py index d0d81962c..993860ef0 100644 --- a/scaler/config/section/native_worker_adapter.py +++ b/scaler/config/section/native_worker_adapter.py @@ -2,11 +2,13 @@ from typing import Optional, Tuple from scaler.config import defaults +from scaler.config.mixins import config_section from scaler.config.types.object_storage_server import ObjectStorageConfig from scaler.config.types.worker import WorkerCapabilities from scaler.config.types.zmq import ZMQConfig +@config_section @dataclasses.dataclass class NativeWorkerAdapterConfig: scheduler_address: ZMQConfig diff --git a/scaler/config/section/object_storage_server.py b/scaler/config/section/object_storage_server.py index e8e56fd61..ede247b35 100644 --- a/scaler/config/section/object_storage_server.py +++ b/scaler/config/section/object_storage_server.py @@ -1,8 +1,10 @@ import dataclasses +from scaler.config.mixins import config_section from scaler.config.types.object_storage_server import ObjectStorageConfig +@config_section @dataclasses.dataclass class ObjectStorageServerConfig: object_storage_address: ObjectStorageConfig diff --git a/scaler/config/section/scheduler.py b/scaler/config/section/scheduler.py index 50518ce92..07c3ecfc5 100644 --- a/scaler/config/section/scheduler.py +++ b/scaler/config/section/scheduler.py @@ -3,6 +3,7 @@ from urllib.parse import urlparse from scaler.config import defaults +from scaler.config.mixins import config_section from scaler.config.types.object_storage_server import ObjectStorageConfig from scaler.config.types.zmq import ZMQConfig from scaler.scheduler.allocate_policy.allocate_policy import AllocatePolicy @@ -10,6 +11,7 @@ from scaler.utility.logging.utility import LoggingLevel +@config_section @dataclasses.dataclass class SchedulerConfig: scheduler_address: ZMQConfig = dataclasses.field() diff --git a/scaler/config/section/symphony_worker_adapter.py b/scaler/config/section/symphony_worker_adapter.py index c05ff211c..4d58619b6 100644 --- a/scaler/config/section/symphony_worker_adapter.py +++ b/scaler/config/section/symphony_worker_adapter.py @@ -2,12 +2,14 @@ from typing import Optional, Tuple from scaler.config import defaults +from scaler.config.mixins import config_section from scaler.config.types.object_storage_server import ObjectStorageConfig from scaler.config.types.worker import WorkerCapabilities from scaler.config.types.zmq import ZMQConfig from scaler.utility.logging.utility import LoggingLevel +@config_section @dataclasses.dataclass class SymphonyWorkerConfig: scheduler_address: ZMQConfig diff --git a/scaler/config/section/top.py b/scaler/config/section/top.py index 32bad8a83..d36f32acb 100644 --- a/scaler/config/section/top.py +++ b/scaler/config/section/top.py @@ -1,8 +1,10 @@ import dataclasses +from scaler.config.mixins import config_section from scaler.config.types.zmq import ZMQConfig +@config_section @dataclasses.dataclass class TopConfig: monitor_address: ZMQConfig diff --git a/scaler/config/section/webui.py b/scaler/config/section/webui.py index a9697ed62..e2f9482cd 100644 --- a/scaler/config/section/webui.py +++ b/scaler/config/section/webui.py @@ -2,9 +2,11 @@ from typing import Optional, Tuple from scaler.config import defaults +from scaler.config.mixins import config_section from scaler.config.types.zmq import ZMQConfig +@config_section @dataclasses.dataclass class WebUIConfig: monitor_address: ZMQConfig