Merge branch 'main' into fix/redshift-single-column-sortkey

Author: simen-strand

docs/guides/linter.md

@@ -126,6 +126,14 @@ Error: Linter detected errors in the code. Please fix them before proceeding.
 
 Use `sqlmesh lint --help` for more information.
 
+You can pass `--local` to run lint without loading state from the configured state connection:
+
+``` bash
+$ sqlmesh lint --local
+```
+
+This can make linting faster in repositories where all referenced models are loaded from local files. In multi-repository setups, or when linting only a subset of projects, `--local` may cause additional linting errors because SQLMesh will not resolve references or schemas from models that exist only in remote state.
+
 
 ## Applying linting rules
 
@@ -258,4 +266,4 @@ You may specify that a rule's violation should not error and only log a warning
     )
     ```
 
-SQLMesh will raise an error if the same rule is included in more than one of the `rules`, `warn_rules`, and `ignored_rules` keys since they should be mutually exclusive.
+SQLMesh will raise an error if the same rule is included in more than one of the `rules`, `warn_rules`, and `ignored_rules` keys since they should be mutually exclusive.

docs/reference/cli.md

@@ -650,6 +650,9 @@ Usage: sqlmesh lint [OPTIONS]
 
 Options:
   --model TEXT           A model to lint. Multiple models can be linted.  If no models are specified, every model will be linted.
+  --local                Lint using only locally loaded project files without loading state. In multi-repository setups, or when
+                         linting only a subset of projects, this may cause additional linting errors because SQLMesh will not resolve
+                         references or schemas from models that exist only in remote state.
   --help                 Show this message and exit.
 
-```
+```

docs/reference/configuration.md

@@ -114,7 +114,7 @@ Formatting settings for the `sqlmesh format` command and UI.
 | `normalize`           | Whether to normalize SQL (Default: False)                                                      | boolean |    N     |
 | `pad`                 | The number of spaces to use for padding (Default: 2)                                           |   int   |    N     |
 | `indent`              | The number of spaces to use for indentation (Default: 2)                                       |   int   |    N     |
-| `normalize_functions` | Whether to normalize function names. Supported values are: 'upper' and 'lower' (Default: None) | string  |    N     |
+| `normalize_functions` | How to normalize function name casing. `false` (default) preserves the casing of custom and audit function names as written; `"upper"` uppercases all function names; `"lower"` lowercases all function names; `true` defers to SQLGlot's generator default and uppercases all function names including custom ones; `null` (or omitting the key) is excluded during serialization and therefore takes the same `false` default path — it does **not** defer to SQLGlot's generator default. Note: SQLGlot built-in function names may still be canonicalized by the parser regardless of this setting. | string \| boolean \| null |    N     |
 | `leading_comma`       | Whether to use leading commas (Default: False)                                                 | boolean |    N     |
 | `max_text_width`      | The maximum text width in a segment before creating new lines (Default: 80)                    |   int   |    N     |
 | `append_newline`      | Whether to append a newline to the end of the file (Default: False)                            | boolean |    N     |

sqlmesh/cli/main.py

@@ -44,6 +44,17 @@
 LOCAL_ONLY_COMMANDS = ("format",)
 
 
+class _SQLMeshGroup(click.Group):
+    def parse_args(self, ctx: click.Context, args: t.List[str]) -> t.List[str]:
+        rest = super().parse_args(ctx, args)
+        # Preserve the subcommand arguments because Click consumes them before invoking the group callback.
+        protected_args = getattr(ctx, "_protected_args", None)
+        if protected_args is None:
+            protected_args = ctx.protected_args
+        ctx.meta["subcommand_args"] = tuple(protected_args) + tuple(ctx.args)
+        return rest
+
+
 def _sqlmesh_version() -> str:
     try:
         from sqlmesh import __version__
@@ -53,7 +64,7 @@ def _sqlmesh_version() -> str:
         return "0.0.0"
 
 
-@click.group(no_args_is_help=True)
+@click.group(cls=_SQLMeshGroup, no_args_is_help=True)
 @click.version_option(version=_sqlmesh_version(), message="%(version)s")
 @opt.paths
 @opt.config
@@ -118,6 +129,9 @@ def cli(
     load = True
     # Local-only gating must hold for any number of --paths, so it stays outside the block below.
     load_state = ctx.invoked_subcommand not in LOCAL_ONLY_COMMANDS
+    # The parent callback constructs Context before Click invokes `lint`, so inspect its parsed args here.
+    if ctx.invoked_subcommand == "lint" and "--local" in ctx.meta["subcommand_args"]:
+        load_state = False
 
     if len(paths) == 1:
         path = os.path.abspath(paths[0])
@@ -1194,6 +1208,12 @@ def environments(obj: Context) -> None:
     multiple=True,
     help="A model to lint. Multiple models can be linted. If no models are specified, every model will be linted.",
 )
+@click.option(
+    "--local",
+    is_flag=True,
+    expose_value=False,
+    help="Lint using only locally loaded project files without loading state.",
+)
 @click.pass_obj
 @error_handler
 @cli_analytics

sqlmesh/core/config/format.py

@@ -12,7 +12,21 @@ class FormatConfig(BaseConfig):
         normalize: Whether to normalize the SQL code or not.
         pad: The number of spaces to use for padding.
         indent: The number of spaces to use for indentation.
-        normalize_functions: Whether or not to normalize all function names. Possible values are: 'upper', 'lower'
+        normalize_functions: How to normalize function name casing.
+
+            * ``False`` (default) — preserves the original spelling of custom and audit
+              function names.  SQLGlot built-in functions (e.g. ``COUNT``, ``SUM``) may
+              still be uppercased because the parser discards the original token.
+            * ``"upper"`` — uppercases all function names, including custom audit
+              references.
+            * ``"lower"`` — lowercases all function names, including built-in ones.
+            * ``True`` — defers to SQLGlot's generator default, which uppercases all
+              function names including custom ones.
+            * ``None`` — excluded from the serialized generator options by Pydantic's
+              ``exclude_none`` behaviour, so ``format_model_expressions`` falls back to
+              its own ``False`` default.  Setting this in YAML as ``null`` or omitting
+              the key is therefore equivalent to ``false``; it does **not** defer to
+              SQLGlot's generator default the way ``True`` does.
         leading_comma: Whether to use leading commas or not.
         max_text_width: The maximum text width in a segment before creating new lines.
         append_newline: Whether to append a newline to the end of the file or not.
@@ -22,7 +36,7 @@ class FormatConfig(BaseConfig):
     normalize: bool = False
     pad: int = 2
     indent: int = 2
-    normalize_functions: t.Optional[str] = None
+    normalize_functions: t.Union[str, bool, None] = False
     leading_comma: bool = False
     max_text_width: int = 80
     append_newline: bool = False

sqlmesh/core/config/root.py

@@ -76,6 +76,24 @@ def validate_regex_key_dict(value: t.Dict[str | re.Pattern, t.Any]) -> t.Dict[re
     return compile_regex_mapping(value)
 
 
+def _canonicalize(obj: object) -> object:
+    """Recursively convert an object into a canonical, order-stable form for hashing.
+
+    ``set``/``frozenset`` iteration order is not stable across Python processes, so
+    pickling them directly yields non-deterministic bytes. That makes any hash derived
+    from the pickle (e.g. ``Config.fingerprint``) change run-to-run, which silently
+    invalidates on-disk caches keyed by the fingerprint. Sorting set members into a
+    list restores determinism while preserving contents.
+    """
+    if isinstance(obj, (set, frozenset)):
+        return sorted(map(_canonicalize, obj))  # type: ignore[type-var]
+    if isinstance(obj, dict):
+        return {k: _canonicalize(v) for k, v in obj.items()}
+    if isinstance(obj, (list, tuple)):
+        return type(obj)(map(_canonicalize, obj))
+    return obj
+
+
 if t.TYPE_CHECKING:
     from sqlmesh.core._typing import Self
 
@@ -364,4 +382,8 @@ def dialect(self) -> t.Optional[str]:
 
     @property
     def fingerprint(self) -> str:
-        return str(zlib.crc32(pickle.dumps(self.dict(exclude={"loader", "notification_targets"}))))
+        return str(
+            zlib.crc32(
+                pickle.dumps(_canonicalize(self.dict(exclude={"loader", "notification_targets"})))
+            )
+        )

sqlmesh/core/dialect.py

@@ -790,6 +790,7 @@ def format_model_expressions(
     expressions: t.List[exp.Expr],
     dialect: t.Optional[str] = None,
     rewrite_casts: bool = True,
+    normalize_functions: t.Union[str, bool, None] = False,
     **kwargs: t.Any,
 ) -> str:
     """Format a model's expressions into a standardized format.
@@ -798,6 +799,21 @@ def format_model_expressions(
         expressions: The model's expressions, must be at least model def + query.
         dialect: The dialect to render the expressions as.
         rewrite_casts: Whether to rewrite all casts to use the :: syntax.
+        normalize_functions: How to normalize function name casing.
+
+            * ``False`` (default) — preserves the original spelling of custom and audit
+              function names.  SQLGlot built-in functions may still canonicalize because
+              the parser discards the original token.
+            * ``"upper"`` — uppercases all function names including custom audit
+              references.
+            * ``"lower"`` — lowercases all function names including built-ins.
+            * ``True`` — defers to SQLGlot's generator default (uppercase).
+            * ``None`` — passes ``None`` directly to the SQLGlot generator, which
+              defers to SQLGlot's own default (typically uppercase, but may vary by
+              dialect).  Note: this is the **direct generator API** behaviour.  When
+              called via ``FormatConfig``, ``None`` is excluded by Pydantic's
+              ``exclude_none`` serialization and this function receives its own ``False``
+              default instead — so the two paths are not equivalent.
         **kwargs: Additional keyword arguments to pass to the sql generator.
 
     Returns:
@@ -807,7 +823,9 @@ def format_model_expressions(
         # Meta expressions (MODEL/AUDIT/METRIC) are SQLMesh DDL, not standard SQL,
         # so they must never be transpiled to the target dialect (e.g. tsql would
         # rewrite a boolean property like `allow_partials TRUE` to `(1 = 1)`).
-        return expressions[0].sql(pretty=True, dialect=None)
+        return expressions[0].sql(
+            pretty=True, dialect=None, normalize_functions=normalize_functions
+        )
 
     if rewrite_casts:
 
@@ -844,6 +862,7 @@ def cast_to_colon(node: exp.Expr) -> exp.Expr:
         expression.sql(
             pretty=True,
             dialect=None if is_meta_expression(expression) else dialect,
+            normalize_functions=normalize_functions,
             **kwargs,
         )
         for expression in expressions

sqlmesh/core/macros.py

@@ -789,7 +789,7 @@ def filter_(evaluator: MacroEvaluator, *args: t.Any) -> t.List[t.Any]:
     """
     *items, func = args
     items, func = _norm_var_arg_lambda(evaluator, func, *items)  # type: ignore
-    return list(filter(lambda arg: evaluator.eval_expression(func(arg)), items))
+    return list(filter(lambda arg: evaluator.eval_expression(func(arg)), ensure_collection(items)))
 
 
 def _optional_expression(

tests/cli/test_cli.py

@@ -2305,6 +2305,24 @@ def test_lint_still_loads_state(runner: CliRunner, tmp_path: Path, mocker):
     assert mock.called, "state-sync was never accessed during `lint`"
 
 
+def test_lint_local_runs_without_state(runner: CliRunner, tmp_path: Path, mocker):
+    mock = _setup_local_only_project(tmp_path, mocker)
+    init_spy = mocker.spy(Context, "__init__")
+
+    result = runner.invoke(cli, ["--paths", str(tmp_path), "lint", "--local"])
+
+    assert result.exit_code == 0, f"Lint failed: {result.output}\nException: {result.exception}"
+    assert init_spy.called, "Context was never constructed"
+    for call in init_spy.call_args_list:
+        assert "load_state" in call.kwargs, (
+            "CLI didn't pass load_state= explicitly; missing kwarg defaults to True silently"
+        )
+        assert call.kwargs["load_state"] is False, (
+            f"Context was constructed with load_state={call.kwargs['load_state']} for `lint --local`"
+        )
+    mock.assert_not_called()
+
+
 @pytest.mark.parametrize("command", ["format"])
 def test_local_only_commands_skip_state_multiple_paths(
     runner: CliRunner, tmp_path: Path, mocker, command: str

tests/core/test_config.py

@@ -1574,3 +1574,66 @@ def test_load_configs_in_dbt_project_without_config_py(tmp_path: Path):
     # model_defaults
     assert config.model_defaults.dialect == "duckdb"  # from dbt profiles.yml
     assert config.model_defaults.start == "2020-01-01"  # from sqlmesh.yaml
+
+
+def test_canonicalize_sorts_sets() -> None:
+    from sqlmesh.core.config.root import _canonicalize
+
+    assert _canonicalize({3, 1, 2}) == [1, 2, 3]
+    assert _canonicalize(frozenset(["b", "a", "c"])) == ["a", "b", "c"]
+
+
+def test_canonicalize_recurses_into_containers() -> None:
+    from sqlmesh.core.config.root import _canonicalize
+
+    assert _canonicalize({"rules": {"z", "a"}, "nested": [{3, 1}, ("x", {"q", "b"})]}) == {
+        "rules": ["a", "z"],
+        "nested": [[1, 3], ("x", ["b", "q"])],
+    }
+
+
+def test_canonicalize_preserves_non_set_values_and_types() -> None:
+    from sqlmesh.core.config.root import _canonicalize
+
+    assert _canonicalize({"a": 1, "b": [1, 2, 3]}) == {"a": 1, "b": [1, 2, 3]}
+    # list/tuple types are preserved (not coerced into each other)
+    assert isinstance(_canonicalize((1, 2)), tuple)
+    assert isinstance(_canonicalize([1, 2]), list)
+
+
+def test_config_fingerprint_is_deterministic_across_processes() -> None:
+    """Config.fingerprint keys the on-disk model cache and must be stable across runs.
+
+    Set/frozenset iteration order depends on PYTHONHASHSEED, so a config containing a
+    set-valued field (e.g. linter.rules) would otherwise hash differently in each
+    process, silently invalidating the cache. Run the same config in two subprocesses
+    with different hash seeds and assert the fingerprint matches.
+    """
+    import subprocess
+    import sys
+
+    program = (
+        "from sqlmesh.core.config import Config, ModelDefaultsConfig\n"
+        "from sqlmesh.core.config.linter import LinterConfig\n"
+        "config = Config(\n"
+        "    model_defaults=ModelDefaultsConfig(dialect='duckdb'),\n"
+        "    linter=LinterConfig(\n"
+        "        enabled=True,\n"
+        "        rules={'ruleA', 'ruleB', 'ruleC', 'ruleD', 'ruleE', 'ruleF'},\n"
+        "    ),\n"
+        ")\n"
+        "print(config.fingerprint)\n"
+    )
+
+    def _fingerprint(hashseed: str) -> str:
+        env = {**os.environ, "PYTHONHASHSEED": hashseed}
+        result = subprocess.run(
+            [sys.executable, "-c", program],
+            capture_output=True,
+            text=True,
+            env=env,
+            check=True,
+        )
+        return result.stdout.strip()
+
+    assert _fingerprint("0") == _fingerprint("12345")