chore: module scope should not require the app context

.gitattributes

@@ -1 +1,2 @@
 docker/**/*.sh text eol=lf
+docs/static/img/erd.svg -diff

.github/workflows/python-importable.yml

@@ -0,0 +1,27 @@
+name: pre-commit checks
+
+on:
+  push:
+    branches:
+      - "master"
+      - "[0-9].[0-9]"
+  pull_request:
+    types: [synchronize, opened, reopened, ready_for_review]
+
+# cancel previous workflow jobs for PRs
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  test_python_imports:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: ./.github/actions/setup-backend/
+      - name: Enable brew and helm-docs
+        run: |
+          echo "Checking that all python modules are importable"
+          python scripts/test_imports.py superset

UPDATING.md

@@ -47,6 +47,10 @@ assists people when migrating to a new version.
   more clearly provides access to all databases, as specified in its name. Before it only allowed
   listing all databases in CRUD-view and dropdown and didn't provide access to data as it
   seemed the name would imply.
+- [28483](https://github.com/apache/superset/pull/28483) Starting with this version we bundle
+  translations inside the python package. This includes the .mo files needed by pybabel on the
+  backend, as well as the .json files used by the frontend. If you were doing anything before
+  as part of your bundling to expose translation packages, it's probably not needed anymore.
 
 ### Potential Downtime
 

docs/docs/contributing/development.mdx

@@ -208,7 +208,7 @@ gunicorn "superset.app:create_app()" -k "geventwebsocket.gunicorn.workers.Gevent
 You can log anything to the browser console, including objects:
 
 ```python
-from superset import app
+from flask import current_app as app
 app.logger.error('An exception occurred!')
 app.logger.info(form_data)
 ```

scripts/benchmark_migration.py

@@ -33,7 +33,7 @@
 from sqlalchemy import create_engine, inspect
 from sqlalchemy.ext.automap import automap_base
 
-from superset import db
+from superset.extensions import db
 from superset.utils.mock_data import add_sample_rows
 
 logger = logging.getLogger(__name__)

scripts/erd/erd.py

@@ -29,7 +29,7 @@
 import click
 import jinja2
 
-from superset import db
+from superset.extensions import db
 
 GROUPINGS: dict[str, Iterable[str]] = {
     "Core": [

scripts/messages.pot.tmp

@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.

scripts/test_imports.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""
+Test importability of all modules within a package with parallel
+execution support.
+
+This was implemented to prevent usage of the app's context and configuration
+located at app.config within module scope. It may also identify other issues
+such as circular imports or anything else that may prevent a module from being
+imported independently.
+"""
+
+import argparse
+import os
+import re
+import subprocess
+import sys
+from concurrent.futures import as_completed, ThreadPoolExecutor  # Import as_completed
+from typing import List
+
+EXCLUDE_FILE_PATTERNS: List[str] = [
+    r"^superset/migrations/",
+    r"^tests/integration_tests/migrations/",
+]
+ROOT_FOLDERS = ["superset", "tests"]
+
+
+def test_module_import(file_path: str) -> str | None:
+    """Test if a module can be imported independently"""
+    module_path = file_path.replace(".py", "").replace("/", ".")
+    splitted = module_path.split(".")
+    from_part = ".".join(splitted[:-1])
+    import_part = splitted[-1]
+    import_statement = f"from {from_part} import {import_part}"
+    try:
+        subprocess.run(
+            ["python", "-c", import_statement],
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+        )
+        return None
+    except subprocess.CalledProcessError as e:
+        if e.stderr:
+            return e.stderr.decode("utf-8").strip()
+        return str(e)
+
+
+def get_module_paths(package_path: str) -> list[str]:
+    paths = []
+    for root, dirs, files in os.walk(package_path):
+        for file in files:
+            filepath = os.path.normpath(os.path.join(package_path, root, file))
+            relative_path = os.path.relpath(filepath, package_path)
+            if file.endswith(".py") and all(
+                not re.match(pattern, relative_path)
+                for pattern in EXCLUDE_FILE_PATTERNS
+            ):
+                paths.append(relative_path)
+    return paths
+
+
+def test_import(
+    path_pattern: str | None = None, max_workers: int | None = None
+) -> None:
+    """Test importability of all modules within a package"""
+    error_count = 0
+    processed_count = 0
+    paths = []
+    for folder in ROOT_FOLDERS:
+        paths += get_module_paths(folder)
+    if path_pattern:
+        filtered_path = []
+        for path in paths:
+            if re.match(path_pattern, path):
+                filtered_path.append(path)
+        paths = filtered_path
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {executor.submit(test_module_import, path): path for path in paths}
+        for future in as_completed(futures):  # Use as_completed
+            path = futures[future]
+            processed_count += 1
+            message = f"[{processed_count}/{len(paths)}] {path}"
+            error = future.result()
+            if error:
+                print(f"❌ {message}")
+                print(error)
+                error_count += 1
+            else:
+                print(f"✅ {message}")
+
+    print(f"Total errors: {error_count}")
+    if error_count:
+        sys.exit(1)
+
+
+def parse_arguments() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Test importability of all modules within a package with parallel execution support."
+    )
+    parser.add_argument(
+        "--workers",
+        type=int,
+        default=os.cpu_count(),
+        help="Number of worker threads for parallel execution (default is number of CPU cores)",
+    )
+    parser.add_argument(
+        "path",
+        type=str,
+        default="*",
+        help="Path filter",
+    )
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_arguments()
+    test_import(args.path, args.workers)

superset/__init__.py

@@ -16,16 +16,13 @@
 # under the License.
 """Package's main module!"""
 
-from flask import current_app, Flask
 from werkzeug.local import LocalProxy
 
-from superset.app import create_app  # noqa: F401
 from superset.extensions import (
     appbuilder,  # noqa: F401
     cache_manager,
-    db,  # noqa: F401
     event_logger,  # noqa: F401
-    feature_flag_manager,
+    feature_flag_manager,  # noqa: F401
     manifest_processor,
     results_backend_manager,
     security_manager,  # noqa: F401
@@ -37,15 +34,26 @@
 #  to declare "global" dependencies is to define it in extensions.py,
 #  then initialize it in app.create_app(). These fields will be removed
 #  in subsequent PRs as things are migrated towards the factory pattern
-app: Flask = current_app
 cache = cache_manager.cache
-conf = LocalProxy(lambda: current_app.config)
-get_feature_flags = feature_flag_manager.get_feature_flags
+data_cache = LocalProxy(lambda: cache_manager.data_cache)
 get_manifest_files = manifest_processor.get_manifest_files
-is_feature_enabled = feature_flag_manager.is_feature_enabled
 results_backend = LocalProxy(lambda: results_backend_manager.results_backend)
 results_backend_use_msgpack = LocalProxy(
     lambda: results_backend_manager.should_use_msgpack
 )
-data_cache = LocalProxy(lambda: cache_manager.data_cache)
 thumbnail_cache = LocalProxy(lambda: cache_manager.thumbnail_cache)
+
+__all__ = [
+    "appbuilder",
+    "cache",
+    "cache_manager",
+    "data_cache",
+    "event_logger",
+    "feature_flag_manager",
+    "get_manifest_files",
+    "manifest_processor",
+    "results_backend",
+    "results_backend_manager",
+    "results_backend_use_msgpack",
+    "thumbnail_cache",
+]

superset/advanced_data_type/api.py

@@ -29,9 +29,6 @@
 from superset.extensions import event_logger
 from superset.views.base_api import BaseSupersetApi
 
-config = app.config
-ADVANCED_DATA_TYPES = config["ADVANCED_DATA_TYPES"]
-
 
 class AdvancedDataTypeRestApi(BaseSupersetApi):
     """
@@ -96,7 +93,7 @@ def get(self, **kwargs: Any) -> Response:
         item = kwargs["rison"]
         advanced_data_type = item["type"]
         values = item["values"]
-        addon = ADVANCED_DATA_TYPES.get(advanced_data_type)
+        addon = app.config["ADVANCED_DATA_TYPES"].get(advanced_data_type)
         if not addon:
             return self.response(
                 400,
@@ -148,4 +145,4 @@ def get_types(self) -> Response:
             500:
               $ref: '#/components/responses/500'
         """
-        return self.response(200, result=list(ADVANCED_DATA_TYPES.keys()))
+        return self.response(200, result=list(app.config["ADVANCED_DATA_TYPES"].keys()))

superset/app.py

@@ -28,6 +28,7 @@
 
 def create_app(superset_config_module: Optional[str] = None) -> Flask:
     app = SupersetApp(__name__)
+    print("in CREATE_APP")
 
     try:
         # Allow user to override our config completely

superset/async_events/async_query_manager.py

@@ -20,7 +20,7 @@
 
 import jwt
 import redis
-from flask import Flask, Request, request, Response, session
+from flask import current_app, Flask, Request, request, Response, session
 
 from superset.utils import json
 from superset.utils.core import get_user_id
@@ -121,8 +121,7 @@ def init_app(self, app: Flask) -> None:
         if config["GLOBAL_ASYNC_QUERIES_REGISTER_REQUEST_HANDLERS"]:
             self.register_request_handlers(app)
 
-        # pylint: disable=import-outside-toplevel
-        from superset.tasks.async_queries import (
+        from superset.tasks.async_queries import (  # pylint: disable=import-outside-toplevel
             load_chart_data_into_cache,
             load_explore_json_into_cache,
         )
@@ -191,8 +190,9 @@ def submit_explore_json_job(
         force: Optional[bool] = False,
         user_id: Optional[int] = None,
     ) -> dict[str, Any]:
-        # pylint: disable=import-outside-toplevel
-        from superset import security_manager
+        from superset.extensions import (  # pylint: disable=import-outside-toplevel
+            security_manager,
+        )
 
         job_metadata = self.init_job(channel_id, user_id)
         self._load_explore_json_into_cache_job.delay(
@@ -202,6 +202,7 @@ def submit_explore_json_job(
             form_data,
             response_type,
             force,
+            soft_time_limit=current_app.config["SQLLAB_ASYNC_TIME_LIMIT_SEC"],
         )
         return job_metadata
 
@@ -211,8 +212,9 @@ def submit_chart_data_job(
         form_data: dict[str, Any],
         user_id: Optional[int] = None,
     ) -> dict[str, Any]:
-        # pylint: disable=import-outside-toplevel
-        from superset import security_manager
+        from superset.extensions import (  # pylint: disable=import-outside-toplevel
+            security_manager,
+        )
 
         # if it's guest user, we want to pass the guest token to the celery task
         # chart data cache key is calculated based on the current user
@@ -224,6 +226,7 @@ def submit_chart_data_job(
             if (guest_user := security_manager.get_current_guest_user_if_guest())
             else job_metadata,
             form_data,
+            soft_time_limit=current_app.config["SQLLAB_ASYNC_TIME_LIMIT_SEC"],
         )
         return job_metadata
 

superset/available_domains/api.py

@@ -16,10 +16,9 @@
 # under the License.
 import logging
 
-from flask import Response
+from flask import current_app as app, Response
 from flask_appbuilder.api import expose, protect, safe
 
-from superset import conf
 from superset.available_domains.schemas import AvailableDomainsSchema
 from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP
 from superset.extensions import event_logger
@@ -70,6 +69,6 @@ def get(self) -> Response:
               $ref: '#/components/responses/403'
         """
         result = self.available_domains_schema.dump(
-            {"domains": conf.get("SUPERSET_WEBSERVER_DOMAINS")}
+            {"domains": app.conf.get("SUPERSET_WEBSERVER_DOMAINS")}
         )
         return self.response(200, result=result)