Merge branch 'main' into chore/update-engine-list

Author: cmgoffena13

.github/workflows/pr.yaml

@@ -457,7 +457,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        dbt-version: ['1.3', '1.4', '1.5', '1.6', '1.7', '1.8', '1.9', '1.10']
+        dbt-version: ['1.3', '1.4', '1.5', '1.6', '1.7', '1.8', '1.9', '1.10', '1.11']
     steps:
       - uses: actions/checkout@v7
       - name: Set up Python

Makefile

@@ -33,8 +33,8 @@ install-dev-dbt-%:
 	echo "Installing dbt version: $$version"; \
 	cp pyproject.toml pyproject.toml.backup; \
 	$(SED_INPLACE) 's/"pydantic>=2.0.0"/"pydantic"/g' pyproject.toml; \
-	if [ "$$version" = "1.10.0" ]; then \
-		echo "Applying special handling for dbt 1.10.0"; \
+	if [ "$$version" = "1.10.0" ] || [ "$$version" = "1.11.0" ]; then \
+		echo "Applying special handling for dbt $$version"; \
 		$(SED_INPLACE) -E 's/"(dbt-core)[^"]*"/"\1~='"$$version"'"/g' pyproject.toml; \
 		$(SED_INPLACE) -E 's/"(dbt-(bigquery|duckdb|snowflake|athena-community|clickhouse|redshift|trino))[^"]*"/"\1"/g' pyproject.toml; \
 		$(SED_INPLACE) -E 's/"(dbt-databricks)[^"]*"/"\1~='"$$version"'"/g' pyproject.toml; \

sqlmesh/core/model/meta.py

@@ -321,17 +321,19 @@ def _column_descriptions_validator(
         if isinstance(vs, (exp.Tuple, exp.Array)):
             vs = vs.expressions
 
-        raw_col_descriptions = (
-            vs
+        # Normalize each part while it is still an identifier, so that a quoted
+        # column keeps its case on dialects where quoting makes it significant.
+        col_descriptions = (
+            {normalize_identifiers(k, dialect=dialect).name: v for k, v in vs.items()}
             if isinstance(vs, dict)
-            else {".".join([part.this for part in v.this.parts]): v.expression.name for v in vs}
+            else {
+                ".".join(
+                    normalize_identifiers(part, dialect=dialect).name for part in v.this.parts
+                ): v.expression.name
+                for v in vs
+            }
         )
 
-        col_descriptions = {
-            normalize_identifiers(k, dialect=dialect).name: v
-            for k, v in raw_col_descriptions.items()
-        }
-
         columns_to_types = data.get("columns_to_types_")
         if columns_to_types:
             from sqlmesh.core.console import get_console

tests/core/test_model.py

@@ -1000,6 +1000,75 @@ def test_column_descriptions(sushi_context, assert_exp_eq):
     assert model.column_descriptions == {"id": "primary key", "foo": "bar"}
 
 
+def test_column_descriptions_quoted_identifier():
+    expressions = d.parse(
+        """
+        MODEL (
+            name db.table,
+            kind FULL,
+            dialect snowflake,
+            column_descriptions (
+                "myColumn" = 'a case-sensitive column',
+                other_column = 'an unquoted column'
+            )
+        );
+
+        SELECT 1 AS "myColumn", 2 AS other_column
+    """
+    )
+    model = load_sql_based_model(expressions, dialect="snowflake")
+
+    # A quoted key keeps its case, an unquoted one is still normalized.
+    assert model.column_descriptions == {
+        "myColumn": "a case-sensitive column",
+        "OTHER_COLUMN": "an unquoted column",
+    }
+    assert set(model.column_descriptions) <= set(model.columns_to_types)
+
+
+def test_column_descriptions_dotted_identifier():
+    # A nested field is looked up by its dotted path, so every part normalizes on its own.
+    expressions = d.parse(
+        """
+        MODEL (
+            name db.table,
+            kind FULL,
+            dialect bigquery,
+            column_descriptions (
+                record.`myField` = 'a nested field'
+            )
+        );
+
+        SELECT STRUCT(1 AS `myField`) AS record
+    """
+    )
+    model = load_sql_based_model(expressions, dialect="bigquery")
+
+    assert model.column_descriptions == {"record.myfield": "a nested field"}
+
+    expressions = d.parse(
+        """
+        MODEL (
+            name db.table,
+            kind FULL,
+            dialect snowflake,
+            column_descriptions (
+                nested.field = 'an unquoted path',
+                "MyStruct"."myField" = 'a quoted path'
+            )
+        );
+
+        SELECT 1 AS c
+    """
+    )
+    model = load_sql_based_model(expressions, dialect="snowflake")
+
+    assert model.column_descriptions == {
+        "NESTED.FIELD": "an unquoted path",
+        "MyStruct.myField": "a quoted path",
+    }
+
+
 def test_model_jinja_macro_reference_extraction():
     @macro()
     def test_macro(**kwargs) -> None:

tests/web/test_main.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import threading
-from pathlib import Path
+from pathlib import Path, PureWindowsPath
 
 import pyarrow as pa  # type: ignore
 import pytest
@@ -75,6 +75,32 @@ def test_get_file(client: TestClient, project_tmp_path: Path) -> None:
     }
 
 
+def test_get_file_nested_path_matches_directory_listing(
+    client: TestClient, project_tmp_path: Path
+) -> None:
+    models_dir = project_tmp_path / "models"
+    models_dir.mkdir()
+    (models_dir / "mymodel.sql").write_text("SELECT 1")
+
+    response = client.get("/api/files/models/mymodel.sql")
+    assert response.status_code == 200
+    assert response.json()["path"] == "models/mymodel.sql"
+
+
+def test_get_file_relative_path_uses_posix_separators(tmp_path: Path) -> None:
+    file_path = tmp_path / "models" / "mymodel.sql"
+    file_path.parent.mkdir(parents=True)
+    file_path.write_text("SELECT 1")
+
+    windows_relative = PureWindowsPath("models/mymodel.sql")
+    assert windows_relative.as_posix() == "models/mymodel.sql"
+    assert str(windows_relative) == "models\\mymodel.sql"
+
+    file = _get_file_with_content(file_path, windows_relative.as_posix())
+    assert file.path == "models/mymodel.sql"
+    assert file.path != str(windows_relative)
+
+
 def test_get_file_not_found(client: TestClient) -> None:
     response = client.get("/api/files/not_found.txt")
     assert response.status_code == 404

web/server/api/endpoints/files.py

@@ -37,7 +37,7 @@ def get_file(
     """Get a file, including its contents."""
     try:
         file_path = Path(path)
-        file = _get_file_with_content(settings.project_path / file_path, str(file_path))
+        file = _get_file_with_content(settings.project_path / file_path, file_path.as_posix())
     except FileNotFoundError:
         raise HTTPException(status_code=HTTP_404_NOT_FOUND)
 
@@ -155,7 +155,7 @@ def walk_path(
         return sorted(directories, key=lambda x: x.name), sorted(files, key=lambda x: x.name)
 
     directories, files = walk_path(path)
-    relative_path = str(Path(path).relative_to(settings.project_path))
+    relative_path = Path(path).relative_to(settings.project_path).as_posix()
 
     return models.Directory(
         name=os.path.basename(path),

web/server/watcher.py

@@ -55,7 +55,7 @@ async def watch_project() -> None:
                     changes.append(
                         models.ArtifactChange(
                             change=Change.deleted,
-                            path=str(relative_path),
+                            path=relative_path.as_posix(),
                         )
                     )
                 elif change == Change.added:
@@ -69,9 +69,9 @@ async def watch_project() -> None:
                         models.ArtifactChange(
                             type=models.ArtifactType.file,
                             change=change,
-                            path=str(relative_path),
+                            path=relative_path.as_posix(),
                             file=_get_file_with_content(
-                                settings.project_path / relative_path, str(relative_path)
+                                settings.project_path / relative_path, relative_path.as_posix()
                             ),
                         )
                     )