Merge branch 'main' into fix-postgres-grants-clean

Author: sakshipatil-hue

.github/workflows/pr.yaml

@@ -50,7 +50,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: '3.10'
       - name: Install uv
@@ -83,7 +83,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install uv
@@ -135,7 +135,7 @@ jobs:
       - name: Install make
         run: choco install make which -y
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: '3.12'
       - name: Install SQLMesh dev dependencies
@@ -172,7 +172,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: '3.10'
       - name: Install uv
@@ -194,16 +194,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
-      - uses: actions/setup-node@v6
+      - uses: actions/setup-node@v7
         with:
           node-version: '22'
-      - uses: pnpm/action-setup@v4
+      - uses: pnpm/action-setup@v6
         with:
           version: latest
       - name: Get pnpm store directory
         id: pnpm-cache
         run: echo "store=$(pnpm store path)" >> $GITHUB_OUTPUT
-      - uses: actions/cache@v5
+      - uses: actions/cache@v6
         with:
           path: ${{ steps.pnpm-cache.outputs.store }}
           key: pnpm-store-${{ hashFiles('pnpm-lock.yaml') }}
@@ -260,7 +260,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: '3.12'
       - name: Install uv
@@ -344,7 +344,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: '3.12'
       - name: Install uv
@@ -399,10 +399,10 @@ jobs:
     if: false
     steps:
       - uses: actions/checkout@v7
-      - uses: actions/setup-node@v6
+      - uses: actions/setup-node@v7
         with:
           node-version: '22'
-      - uses: pnpm/action-setup@v4
+      - uses: pnpm/action-setup@v6
         with:
           version: latest
       - name: Install dependencies
@@ -416,16 +416,16 @@ jobs:
     if: false
     steps:
       - uses: actions/checkout@v7
-      - uses: actions/setup-node@v6
+      - uses: actions/setup-node@v7
         with:
           node-version: '22'
-      - uses: pnpm/action-setup@v4
+      - uses: pnpm/action-setup@v6
         with:
           version: latest
       - name: Install dependencies
         run: pnpm install
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: '3.12'
       - name: Install uv
@@ -461,7 +461,7 @@ jobs:
     steps:
       - uses: actions/checkout@v7
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: '3.10'
       - name: Install uv

.github/workflows/release.yaml

@@ -10,10 +10,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v7
-      - uses: actions/setup-node@v6
+      - uses: actions/setup-node@v7
         with:
           node-version: '22'
-      - uses: pnpm/action-setup@v4
+      - uses: pnpm/action-setup@v6
         with:
           version: latest
       - name: Install dependencies
@@ -38,7 +38,7 @@ jobs:
           name: ui-dist
           path: web/client/dist/
       - name: Set up Python
-        uses: actions/setup-python@v6
+        uses: actions/setup-python@v7
         with:
           python-version: '3.10'
       - name: Install uv
@@ -65,7 +65,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Create release on GitHub
-        uses: softprops/action-gh-release@v2
+        uses: softprops/action-gh-release@v3
         with:
           generate_release_notes: true
           tag_name: ${{ github.ref_name }}

.github/workflows/release_extension.yaml

@@ -28,11 +28,11 @@ jobs:
           fi
           echo "Version format is valid: $version"
       - name: Setup Node.js
-        uses: actions/setup-node@v6
+        uses: actions/setup-node@v7
         with:
           node-version: '22'
       - name: Install pnpm
-        uses: pnpm/action-setup@v4
+        uses: pnpm/action-setup@v6
         with:
           version: 10
       - name: Install dependencies

.github/workflows/release_shared_js.yaml

@@ -31,7 +31,7 @@ jobs:
           fi
           echo "Version format is valid: $version"
       - name: Setup Node.js
-        uses: actions/setup-node@v6
+        uses: actions/setup-node@v7
         with:
           node-version: '22'
           registry-url: 'https://registry.npmjs.org'
@@ -40,7 +40,7 @@ jobs:
       - name: Print npm version
         run: npm --version
       - name: Install pnpm
-        uses: pnpm/action-setup@v4
+        uses: pnpm/action-setup@v6
         with:
           version: 10
       - name: Install dependencies

docs/guides/configuration.md

@@ -329,6 +329,16 @@ By default, the SQLMesh cache is stored in a `.cache` directory within your proj
 
 The cache directory is automatically created if it doesn't exist. You can clear the cache using the `sqlmesh clean` command.
 
+#### Project index
+
+The `--use-project-index` option on supported commands maintains a persistent model dependency index in the cache directory. Each project writes a file named `<project>_<hash>_model_index.json`.
+
+A full project load with the option enabled creates or refreshes the index. SQLMesh invalidates it when relevant configuration, gateway, macro, audit, or signal metadata changes, or when the set of model files changes. If the index is missing, invalid, or stale, SQLMesh safely falls back to a full project load and rebuilds it.
+
+For operations targeting selected models, the index allows SQLMesh to load only those models and their upstream dependencies.
+
+In multi-repository projects, dependencies that cross project boundaries may not be represented by an individual project's index. SQLMesh detects incomplete scoped loads and falls back to loading the full configured project set.
+
 ### Table/view storage locations
 
 SQLMesh creates schemas, physical tables, and views in the data warehouse/engine. Learn more about why and how SQLMesh creates schema in the ["Why does SQLMesh create schemas?" FAQ](../faq/faq.md#schema-question).

docs/guides/linter.md

@@ -135,6 +135,21 @@ $ sqlmesh lint --local
 
 This can make linting faster in repositories where all referenced models are loaded from local files. In multi-repository setups, or when linting only a subset of projects, `--local` may cause additional linting errors because SQLMesh will not resolve references or schemas from models that exist only in remote state.
 
+For faster targeted linting, enable the persistent project index with `--use-project-index`. When
+models are selected with `--model`, SQLMesh loads, resolves, and validates only those models and
+their transitive upstream dependencies. The same behavior can be enabled by default for the
+Python API and CLI with the `linter.use_project_index` configuration option:
+
+```yaml
+linter:
+  enabled: true
+  use_project_index: true
+```
+
+`Context.lint_models` uses this configuration value when `use_project_index` is omitted. Passing
+`use_project_index=False` explicitly disables it for that call. If a context was already loaded,
+an indexed lint of selected models reloads the context so the requested scope is applied.
+
 
 ## Applying linting rules
 

docs/reference/cli.md

@@ -650,9 +650,12 @@ Usage: sqlmesh lint [OPTIONS]
 
 Options:
   --model TEXT           A model to lint. Multiple models can be linted.  If no models are specified, every model will be linted.
+  --use-project-index    Use the persistent project index. With --model, only the selected models and their upstream dependencies
+                         are loaded, resolved, and validated, so errors in unrelated models are not reported. Without --model,
+                         every model is still loaded and linted.
   --local                Lint using only locally loaded project files without loading state. In multi-repository setups, or when
                          linting only a subset of projects, this may cause additional linting errors because SQLMesh will not resolve
                          references or schemas from models that exist only in remote state.
   --help                 Show this message and exit.
 
-```
+```

docs/reference/configuration.md

@@ -60,6 +60,13 @@ The `model_defaults` key is **required** and must contain a value for the `diale
 
 See all the keys allowed in `model_defaults` at the [model configuration reference page](./model_configuration.md#model-defaults).
 
+### Linter
+
+| Option                    | Description                                                                                                                                        | Type    | Required |
+|---------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|---------|----------|
+| `linter.enabled`          | Whether linting is enabled (Default: `False`)                                                                                                      | boolean | N        |
+| `linter.use_project_index` | Whether to use the persistent project index for linting. Targeted linting loads selected models and their upstream dependencies. (Default: `False`) | boolean | N        |
+
 ### Variables
 
 The `variables` key can be used to provide values for user-defined variables, accessed using the [`@VAR` macro function](../concepts/macros/sqlmesh_macros.md#global-variables) in SQL model definitions, [`context.var` method](../concepts/models/python_models.md#global-variables) in Python model definitions, and [`evaluator.var` method](../concepts/macros/sqlmesh_macros.md#accessing-global-variable-values) in Python macro functions.

sqlmesh/cli/main.py

@@ -141,6 +141,10 @@ def cli(
         if ctx.invoked_subcommand in SKIP_LOAD_COMMANDS:
             load = False
 
+    # Unlike the other commands above, lint can scope its own load for multi-project contexts.
+    if ctx.invoked_subcommand == "lint":
+        load = False
+
     configs = load_configs(config, Context.CONFIG_TYPE, paths, dotenv_path=dotenv)
     log_limit = list(configs.values())[0].log_limit
 
@@ -1209,6 +1213,12 @@ def environments(obj: Context) -> None:
     multiple=True,
     help="A model to lint. Multiple models can be linted. If no models are specified, every model will be linted.",
 )
+@click.option(
+    "--use-project-index",
+    is_flag=True,
+    default=None,
+    help="Use the persistent project index. With --model, only the selected models and their upstream dependencies are loaded, resolved, and validated, so errors in unrelated models are not reported. Without --model, every model is still loaded and linted. Can also be enabled with linter.use_project_index.",
+)
 @click.option(
     "--local",
     is_flag=True,
@@ -1221,9 +1231,18 @@ def environments(obj: Context) -> None:
 def lint(
     obj: Context,
     models: t.Iterator[str],
+    use_project_index: t.Optional[bool],
 ) -> None:
     """Run the linter for the target model(s)."""
-    obj.lint_models(models)
+    obj.lint_models(
+        models,
+        use_project_index=use_project_index,
+    )
+
+    if not obj.models:
+        raise click.ClickException(
+            f"`{obj.path}` doesn't seem to have any models... cd into the proper directory or specify the path(s) with -p."
+        )
 
 
 @cli.group(no_args_is_help=True)

sqlmesh/core/config/linter.py

@@ -16,13 +16,16 @@ class LinterConfig(BaseConfig):
     Args:
         enabled: Flag indicating whether the linter should run
 
+        use_project_index: Whether to use the persistent project index when linting.
+
         rules: A list of error rules to be applied on model
         warn_rules: A list of rules to be applied on models but produce warnings instead of raising errors.
         ignored_rules: A list of rules to be excluded/ignored
 
     """
 
     enabled: bool = False
+    use_project_index: bool = False
 
     rules: t.Set[str] = set()
     warn_rules: t.Set[str] = set()