Make export usable on large accounts (10k SHOW cap)

docs/cli.md

@@ -105,6 +105,7 @@ snowcap export --all
 | `--exclude <types>` | Exclude resource types (used with --all) |
 | `--out <file>` | Write exported config to a file |
 | `--format [json\|yml]` | Output format (default: yml) |
+| `--use-account-usage / --no-use-account-usage` | Read from `SNOWFLAKE.ACCOUNT_USAGE` instead of `SHOW` (default: on). `SHOW ... IN ACCOUNT` is capped at 10,000 rows; `ACCOUNT_USAGE` lags live state by up to ~2 hours |
 
 **Examples:**
 

snowcap/blueprint.py

@@ -1666,20 +1666,22 @@ def fetch_remote_state(self, session, manifest: Manifest) -> State:
         if self._config.sync_resources:
             urns = [item for item in manifest.urns if item.resource_type not in self._config.sync_resources]
             for resource_type in self._config.sync_resources:
-                # Future grants are read in full whenever grants are synced, and neither
-                # the query nor the set of roles queried is narrowed to what the manifest
-                # declares. Narrowing would be sound for a plan that only creates, but
-                # syncing a resource type means removing what is not declared, and a future
-                # grant absent from config is precisely what has to be found.
-                #
-                # Skipping the query when the manifest declared no future grants kept the
-                # ones already in Snowflake out of remote state, so sync could not propose
-                # dropping them -- unseen rather than deliberately kept, with nothing in
-                # the plan to say so. Migrating a config from ALL plus FUTURE pairs to
-                # inherited grants removes the last future grant and hit exactly that: 26
-                # orphaned future grants, zero drops, no warning.
-                list_kwargs: dict[str, Any] = {}
+                # Sync drops whatever remote state omits, so the listing must read the
+                # source the config asked for. list_resource drops unsupported kwargs.
+                list_kwargs: dict[str, Any] = {"use_account_usage": self._config.use_account_usage}
                 if resource_type == ResourceType.GRANT:
+                    # Future grants are read in full whenever grants are synced, and neither
+                    # the query nor the set of roles queried is narrowed to what the manifest
+                    # declares. Narrowing would be sound for a plan that only creates, but
+                    # syncing a resource type means removing what is not declared, and a future
+                    # grant absent from config is precisely what has to be found.
+                    #
+                    # Skipping the query when the manifest declared no future grants kept the
+                    # ones already in Snowflake out of remote state, so sync could not propose
+                    # dropping them -- unseen rather than deliberately kept, with nothing in
+                    # the plan to say so. Migrating a config from ALL plus FUTURE pairs to
+                    # inherited grants removes the last future grant and hit exactly that: 26
+                    # orphaned future grants, zero drops, no warning.
                     list_kwargs["include_future_grants"] = True
                 for fqn in data_provider.list_resource(session, resource_label_for_type(resource_type), **list_kwargs):
                     if self._config.scope == BlueprintScope.DATABASE and fqn.database != self._config.database:

snowcap/cli.py

@@ -358,7 +358,15 @@ def apply(
 )
 @click.option("--out", type=str, help="Write exported config to a file", metavar="<filename>")
 @click.option("--format", type=click.Choice(["json", "yml"]), default="yml", help="Output format")
-def export(resources, export_all, exclude_resources, out, format) -> None:
+@click.option(
+    "--use-account-usage/--no-use-account-usage",
+    default=True,
+    help=(
+        "Read from SNOWFLAKE.ACCOUNT_USAGE instead of SHOW. Required past the 10,000-row "
+        "SHOW cap, but lags live state by up to ~2 hours"
+    ),
+)
+def export(resources, export_all, exclude_resources, out, format, use_account_usage) -> None:
     """
     Generate a resource config for existing Snowflake resources
 
@@ -388,9 +396,9 @@ def export(resources, export_all, exclude_resources, out, format) -> None:
 
     resource_config: dict[str, Any] = {}
     if resources:
-        resource_config = export_resources(include=resources)
+        resource_config = export_resources(include=resources, use_account_usage=use_account_usage)
     elif export_all:
-        resource_config = export_resources(exclude=exclude_resources)
+        resource_config = export_resources(exclude=exclude_resources, use_account_usage=use_account_usage)
     else:
         raise
 

snowcap/client.py

@@ -35,10 +35,18 @@
 # Track queries currently being executed to prevent duplicate execution
 _PENDING_QUERIES: dict[tuple[str, str], threading.Event] = {}
 
+# Invalidation callbacks for caches derived from _EXECUTION_CACHE, registered by the modules
+# that index cached rows -- this module cannot import them, they import it.
+_CACHE_RESET_HOOKS: list[Callable[[], None]] = []
+
+
+def register_cache_reset_hook(hook: Callable[[], None]) -> None:
+    _CACHE_RESET_HOOKS.append(hook)
+
 
 def reset_cache():
     """
-    Reset the SQL execution cache.
+    Reset the SQL execution cache and every cache derived from it.
 
     This clears cached query results so subsequent queries will re-execute.
     Note: This does NOT clear ACCOUNT_USAGE caches, which are designed to persist
@@ -47,6 +55,8 @@ def reset_cache():
     """
     global _EXECUTION_CACHE
     _EXECUTION_CACHE = {}
+    for hook in _CACHE_RESET_HOOKS:
+        hook()
 
 
 def execute(