test: split input/output for planner test

src/frontend/planner_test/Cargo.toml

@@ -6,7 +6,6 @@ homepage = { workspace = true }
 keywords = { workspace = true }
 license = { workspace = true }
 repository = { workspace = true }
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [package.metadata.cargo-machete]
 ignored = ["workspace-hack"]
@@ -18,8 +17,10 @@ normal = ["workspace-hack"]
 anyhow = "1"
 backtrace = "0.3.67"
 console = "0.15"
+expect-test = "1"
 futures = { version = "0.3", default-features = false, features = ["alloc"] }
 itertools = "0.10"
+paste = "1"
 risingwave_frontend = { path = ".." }
 risingwave_sqlparser = { path = "../../sqlparser" }
 serde = { version = "1", features = ["derive"] }
@@ -43,13 +44,8 @@ workspace-hack = { path = "../../workspace-hack" }
 libtest-mimic = "0.6"
 tempfile = "3"
 
-[build-dependencies]
-anyhow = "1"
-walkdir = "2"
-
-[[bin]]
-name = "planner-test-apply"
-path = "src/bin/apply.rs"
+[lib]
+test = false
 
 [[test]]
 name = "planner_test_runner"

src/frontend/planner_test/README.md

@@ -5,17 +5,20 @@ Given a sequence of SQL queries as the input, the test runner will check
 the logical operator tree if any, and the physical operator tree if any.
 
 The test data in YAML format is organized under `tests/testdata` folder.
+The inputs are put under `tests/testdata/input`, and the outputs are
+put under `tests/testdata/output`.
 
 ## Examples of Test Cases
 
 ### SELECT as the test case
 
-You can simply write a `SELECT` query in the `sql` field, and using other fields, including `logical_plan` , `stream_plan` , `binder_error` , etc. for the plan under different situations.
+You can simply write a `SELECT` query in the `sql` field, and use `expected_outputs` to specify the plans you want to check, including `logical_plan` , `stream_plan` , `binder_error` , etc.
 
 ```yaml
 - sql: |
     select * from t
-  binder_error: "Item not found: relation \"t\""
+  expected_outputs:
+  - binder_error
 ```
 
 This is a simple test case that validates the binder's behavior on an illegal SQL.
@@ -24,58 +27,37 @@ This is a simple test case that validates the binder's behavior on an illegal SQ
 - sql: |
     create table t (v1 bigint, v2 double precision);
     select * from t;
-  logical_plan: |
-    LogicalProject { exprs: [$0, $1, $2], expr_alias: [None, None, None] }
-      LogicalScan { table: "t", columns: ["_row_id", "v1", "v2"] }
+  expected_outputs:
+  - logical_plan
+  - batch_plan
 ```
 
-If the SQL is valid, then test runner will compare the generated logical operator tree
-with the expected tree.
+If the SQL is valid, then test runner will compare the generated logical plan and batch plan with the expected ones.
 
 ### EXPLAIN as the test case
 
 Alternatively, you can also write an `EXPLAIN` statement in the `sql` field. In this case, the output field is a single `explain_output` .
 
 ```yaml
 - sql: explain select 1;
-  explain_output: |
-    BatchProject { exprs: [1:Int32] }
-    └─BatchValues { rows: [[]] }
+  expected_outputs:
+  - explain_output
 ```
 
 This is helpful when you want to test `EXPLAIN CREATE ...` and `EXPLAIN (options) ...` statements.
 
-## Update Plans
+## Update Outputs
 
-Firstly, we will need to create a placeholder in yaml testcases:
+The outputs can be automatically generated after you update code or add new test cases.
 
-```yaml
-- sql: |
-    create table t1 (v1 int, v2 int);
-    create table t2 (v1 int, v2 int);
-    create table t3 (v1 int, v2 int);
-    select * from t1 join t2 on (t1.v1 = t2.v1) join t3 on (t2.v2 = t3.v2);
-  logical_plan: ""
-  batch_plan: ""
-  stream_plan: ""
-```
-
-Those plans followed the input SQL are expected outputs.
-
-```
-./risedev apply-planner-test
-```
-
-Then we can find the updated tests at `*.apply.yaml` . If everything is okay, you may run:
+you may run:
 
 ```
 ./risedev do-apply-planner-test
 ```
 
 To apply the new test results.
 
-You may use the `before` key to include other testcases by `id` .
-
 ## Run a single test
 
 ```

src/frontend/planner_test/planner_test.toml

@@ -1,69 +1,32 @@
-
-[tasks.update-planner-test]
-description = "Update planner test data"
-private = true
-script = '''
-#!/usr/bin/env bash
-set -e
-cargo run --bin planner-test-apply
-'''
-
-[tasks.apply-planner-test]
-description = "Generate planner test data"
-dependencies = [
-  "update-planner-test"
-]
-script = '''
-#!/usr/bin/env bash
-set -e
-cd src/frontend/planner_test/tests/testdata/
-
-for f in *.apply.yaml
-do
-  diff "$f" "$(basename "$f" .apply.yaml).yaml" || true
-done
-
-echo "If you want to apply the planner test data, run: $(tput setaf 2)./risedev do-apply-planner-test$(tput sgr 0)"
-'''
-category = "RiseDev - Test"
-
 [tasks.do-apply-planner-test]
 description = "Apply planner test data"
-dependencies = [
-  "update-planner-test"
-]
+category = "RiseDev - Test"
+dependencies = ["install-nextest"]
 script = '''
 #!/usr/bin/env bash
 set -e
-cd src/frontend/planner_test/tests/testdata/
 
-for f in *.apply.yaml
-do
-  SOURCE="$(basename $f .apply.yaml).yaml"
-  if [ -f "$SOURCE" ]; then
-    cat <<EOF > temp.apply.yaml
-# This file is automatically generated. See \`src/frontend/planner_test/README.md\` for more information.
-EOF
-    cat "$f" >> temp.apply.yaml
-    mv temp.apply.yaml "$SOURCE"
-  fi
-done
+UPDATE_EXPECT=1 cargo nextest run -p risingwave_planner_test --retries 0
 
-rm *.apply.yaml
+# If there is a file in /output, but no corresponding one in /input, remove it
+for f in $(find src/frontend/planner_test/tests/testdata/output -type f); do
+    if [ ! -f "src/frontend/planner_test/tests/testdata/input/$(basename $f)" ]; then
+        echo "Removing $(tput setaf 4)$f$(tput sgr0), because there is no corresponding input file."
+        rm $f
+    fi
+done
 
 echo "$(tput setaf 2)Diff applied!$(tput sgr 0)"
 echo "Tip: use the alias $(tput setaf 4)./risedev dapt$(tput sgr0)."
 '''
-category = "RiseDev - Test"
 
 [tasks.dapt]
 alias = "do-apply-planner-test"
 
-
 [tasks.run-planner-test]
 description = "Run planner test"
 category = "RiseDev - Test"
-dependencies = ["warn-on-missing-tools"]
+dependencies = ["install-nextest"]
 script = '''
 #!/usr/bin/env bash
 set -e