Merge branch 'master' into DOC-302-new-etl-tutorial

Author: C00ldudeNoonan

.buildkite/dagster-buildkite/dagster_buildkite/package_spec.py

@@ -232,7 +232,7 @@ def build_steps(self) -> List[BuildkiteTopLevelStep]:
     @property
     def requirements(self):
         # First try to infer requirements from the python package
-        package = PythonPackages.get(self.name)
+        package = PythonPackages.get(self.name)  # pyright: ignore[reportArgumentType]
         if package:
             return set.union(package.install_requires, *package.extras_require.values())
 

.buildkite/dagster-buildkite/dagster_buildkite/pipelines/dagster_oss_nightly_pipeline.py

@@ -2,6 +2,7 @@
 
 from dagster_buildkite.package_spec import PackageSpec
 from dagster_buildkite.python_version import AvailablePythonVersion
+from dagster_buildkite.step_builder import BuildkiteQueue
 from dagster_buildkite.steps.packages import (
     build_steps_from_package_specs,
     gcp_creds_extra_cmds,
@@ -28,6 +29,7 @@ def build_dagster_oss_nightly_steps() -> List[BuildkiteStep]:
                 unsupported_python_versions=[
                     AvailablePythonVersion.V3_12,
                 ],
+                always_run_if=lambda: True,
             ),
             PackageSpec(
                 "python_modules/libraries/dagster-k8s",
@@ -41,6 +43,7 @@ def build_dagster_oss_nightly_steps() -> List[BuildkiteStep]:
                     "nightly",
                 ],
                 pytest_extra_cmds=k8s_extra_cmds,
+                always_run_if=lambda: True,
             ),
             PackageSpec(
                 "examples/experimental/dagster-dlift/kitchen-sink",
@@ -52,9 +55,10 @@ def build_dagster_oss_nightly_steps() -> List[BuildkiteStep]:
                     "KS_DBT_CLOUD_ACCESS_URL",
                     "KS_DBT_CLOUD_DISCOVERY_API_URL",
                 ],
+                always_run_if=lambda: True,
             ),
             PackageSpec(
-                "examples/experimental/dagster-airlift/examples/dbt-example",
+                "examples/starlift-demo",
                 name="airlift-demo-live-tests",
                 env_vars=[
                     "KS_DBT_CLOUD_ACCOUNT_ID",
@@ -63,6 +67,20 @@ def build_dagster_oss_nightly_steps() -> List[BuildkiteStep]:
                     "KS_DBT_CLOUD_ACCESS_URL",
                     "KS_DBT_CLOUD_DISCOVERY_API_URL",
                 ],
+                queue=BuildkiteQueue.DOCKER,
+                always_run_if=lambda: True,
+            ),
+            PackageSpec(
+                "integration_tests/test_suites/dagster-azure-live-tests",
+                name="azure-live-tests",
+                env_vars=[
+                    "TEST_AZURE_TENANT_ID",
+                    "TEST_AZURE_CLIENT_ID",
+                    "TEST_AZURE_CLIENT_SECRET",
+                    "TEST_AZURE_STORAGE_ACCOUNT_ID",
+                    "TEST_AZURE_CONTAINER_ID",
+                ],
+                always_run_if=lambda: True,
             ),
         ]
     )

.buildkite/dagster-buildkite/dagster_buildkite/steps/integration.py

@@ -6,6 +6,7 @@
     GCP_CREDS_LOCAL_FILE,
     LATEST_DAGSTER_RELEASE,
 )
+from dagster_buildkite.git import ChangedFiles
 from dagster_buildkite.package_spec import (
     PackageSpec,
     PytestExtraCommandsFunction,
@@ -42,6 +43,7 @@ def build_integration_steps() -> List[BuildkiteStep]:
     steps += build_k8s_suite_steps()
     steps += build_daemon_suite_steps()
     steps += build_auto_materialize_perf_suite_steps()
+    steps += build_azure_live_test_suite_steps()
 
     return steps
 
@@ -165,6 +167,29 @@ def build_auto_materialize_perf_suite_steps():
     )
 
 
+def skip_if_not_azure_commit():
+    """If no dagster-azure files are changed, skip the azure live tests."""
+    return (
+        None
+        if (any("dagster-azure" in str(path) for path in ChangedFiles.all))
+        else "Not a dagster-azure commit"
+    )
+
+
+def build_azure_live_test_suite_steps() -> List[BuildkiteTopLevelStep]:
+    return PackageSpec(
+        os.path.join("integration_tests", "test_suites", "dagster-azure-live-tests"),
+        skip_if=skip_if_not_azure_commit,
+        env_vars=[
+            "TEST_AZURE_TENANT_ID",
+            "TEST_AZURE_CLIENT_ID",
+            "TEST_AZURE_CLIENT_SECRET",
+            "TEST_AZURE_STORAGE_ACCOUNT_ID",
+            "TEST_AZURE_CONTAINER_ID",
+        ],
+    ).build_steps()
+
+
 def daemon_pytest_extra_cmds(version: AvailablePythonVersion, _):
     return [
         "export DAGSTER_DOCKER_IMAGE_TAG=$${BUILDKITE_BUILD_ID}-" + version.value,

.buildkite/dagster-buildkite/dagster_buildkite/steps/packages.py

@@ -272,14 +272,6 @@ def k8s_extra_cmds(version: AvailablePythonVersion, _) -> List[str]:
 # Some Dagster packages have more involved test configs or support only certain Python version;
 # special-case those here
 EXAMPLE_PACKAGES_WITH_CUSTOM_CONFIG: List[PackageSpec] = [
-    PackageSpec(
-        "examples/with_airflow",
-        unsupported_python_versions=[
-            AvailablePythonVersion.V3_10,
-            AvailablePythonVersion.V3_11,
-            AvailablePythonVersion.V3_12,
-        ],
-    ),
     PackageSpec(
         "examples/assets_smoke_test",
     ),
@@ -367,13 +359,10 @@ def k8s_extra_cmds(version: AvailablePythonVersion, _) -> List[str]:
     PackageSpec(
         "examples/experimental/dagster-blueprints",
     ),
-    PackageSpec(
-        "examples/experimental/dagster-airlift",
-    ),
     # Runs against live dbt cloud instance, we only want to run on commits and on the
     # nightly build
     PackageSpec(
-        "examples/experimental/dagster-airlift/examples/dbt-example",
+        "examples/starlift-demo",
         skip_if=skip_if_not_airlift_or_dlift_commit,
         env_vars=[
             "KS_DBT_CLOUD_ACCOUNT_ID",
@@ -385,16 +374,16 @@ def k8s_extra_cmds(version: AvailablePythonVersion, _) -> List[str]:
         timeout_in_minutes=30,
         queue=BuildkiteQueue.DOCKER,
     ),
+    # Federation tutorial spins up multiple airflow instances, slow to run - use docker queue to ensure
+    # beefier instance
     PackageSpec(
-        "examples/experimental/dagster-airlift/examples/perf-harness",
-        always_run_if=has_dagster_airlift_changes,
-    ),
-    PackageSpec(
-        "examples/experimental/dagster-airlift/examples/tutorial-example",
-        always_run_if=has_dagster_airlift_changes,
+        "examples/airlift-federation-tutorial",
+        skip_if=skip_if_not_airlift_or_dlift_commit,
+        timeout_in_minutes=30,
+        queue=BuildkiteQueue.DOCKER,
     ),
     PackageSpec(
-        "examples/experimental/dagster-airlift/examples/kitchen-sink",
+        "examples/airlift-migration-tutorial",
         always_run_if=has_dagster_airlift_changes,
     ),
     PackageSpec(
@@ -740,6 +729,14 @@ def tox_factors_for_folder(tests_folder_name: str) -> List[str]:
             AvailablePythonVersion.V3_12,
         ],
     ),
+    PackageSpec(
+        "python_modules/libraries/dagster-airlift/perf-harness",
+        always_run_if=has_dagster_airlift_changes,
+    ),
+    PackageSpec(
+        "python_modules/libraries/dagster-airlift/kitchen-sink",
+        always_run_if=has_dagster_airlift_changes,
+    ),
     PackageSpec(
         ".buildkite/dagster-buildkite",
         run_pytest=False,

.buildkite/dagster-buildkite/dagster_buildkite/steps/trigger.py

@@ -28,7 +28,7 @@ def build_trigger_step(
     dagster_commit_hash = safe_getenv("BUILDKITE_COMMIT")
     step: TriggerStep = {
         "trigger": pipeline,
-        "label": f":link: {pipeline} from dagster@{dagster_commit_hash[:6]}",
+        "label": f":link: {pipeline} from dagster@{dagster_commit_hash[:10]}",
         "async": async_step,
         "build": {
             "env": env or {},

.buildkite/dagster-buildkite/dagster_buildkite/utils.py

@@ -363,7 +363,7 @@ def skip_if_no_docs_changes():
     if message_contains("NO_SKIP"):
         return None
 
-    if not is_feature_branch(os.getenv("BUILDKITE_BRANCH")):
+    if not is_feature_branch(os.getenv("BUILDKITE_BRANCH")):  # pyright: ignore[reportArgumentType]
         return None
 
     # If anything changes in the docs directory

.dockerignore

@@ -0,0 +1,9 @@
+.git/
+**/.tox/
+**/.yarn/
+**/__pycache__/
+**/*.egg-info/
+**/node_modules/
+**/*.pyc
+**/.next/
+**/venv/

.gitattributes

@@ -1 +1,2 @@
-*.py diff=python
+*.py diff=python
+**/uv.lock linguist-generated

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+# Automatically request docs team for docs PR review
+/docs/              @neverett

.github/ISSUE_TEMPLATE/report_bug.yml

@@ -89,8 +89,4 @@ body:
     attributes:
       label: Message from the maintainers
       description: This form field should be ignored. This is to include a footer message on the generated issue.
-      value: >
-        Impacted by this issue? Give it a 👍! We factor engagement into prioritization.
-        
-        By submitting this issue, you agree to follow Dagster's
-        [Code of Conduct](https://github.com/dagster-io/dagster/blob/master/.github/CODE_OF_CONDUCT.md).
+      value: Impacted by this issue? Give it a 👍! We factor engagement into prioritization.