TIG-3408: Consolidate lookup genny workloads now that feature flag is…

… on by default (mongodb#554)

src/workloads/execution/GraphLookupWithOnlyUnshardedColls.yml

@@ -0,0 +1,106 @@
+SchemaVersion: 2018-07-01
+Owner: "@mongodb/query"
+Description: |
+  This test exercises the behavior of $graphLookup when both the local and foreign collections are
+  unsharded. When one of the collections is sharded, the pipeline should be moved to the
+  GraphLookup workload, so that this file can continue to be tested in an unsharded environment. We
+  may want to compare results across the two files, so the loading stage (and some queries) should
+  be kept as similar as possible.
+
+  Note: this workload runs on replica sets and sharded clusters.
+
+  The workload consists of the following phases:
+    1. Creating an empty sharded collection distributed across all shards in the cluster.
+    2. Populating collections with data.
+    3. Fsync.
+    4. Running $graphLookups.
+
+Actors:
+- Name: LoadGraphLookupData
+  Type: Loader
+  Threads: 1
+  Phases:
+  - Repeat: 1
+    BatchSize: 3000
+    Threads: 1
+    DocumentCount: &NumDocs 3000
+    Database: &Database test
+    CollectionCount: 3  # Loader will populate 'Collection0' and 'Collection1'.
+    Document:
+      a: {^RandomInt: {min: 1, max: 100}}
+      b: {^RandomInt: {min: 1, max: 100}}
+      c: {^RandomInt: {min: 1, max: 10}}
+      matchOne: {^RandomInt: {min: 1, max: 3000}}
+      secondMatchOne: {^RandomInt: {min: 1, max: 3000}}
+      matchTwo: {^RandomInt: {min: 1, max: 1500}}
+      matchFive: {^RandomInt: {min: 1, max: 600}}
+      secondMatchFive: {^RandomInt: {min: 1, max: 600}}
+      matchTen: {^RandomInt: {min: 1, max: 300}}
+      matchOneHundred: {^RandomInt: {min: 1, max: 30}}
+  - &Nop {Nop: true}
+  - *Nop
+
+- Name: Quiesce
+  Type: RunCommand
+  Threads: 1
+  Phases:
+  - *Nop
+  - Repeat: 1
+    Database: admin
+    Operations:
+    - OperationName: RunCommand
+      OperationCommand:
+        fsync: 1
+  - *Nop
+
+- Name: RunGraphLookups
+  Type: RunCommand
+  Database: *Database
+  Phases:
+  - *Nop
+  - *Nop
+  - Repeat: 10
+    Database: *Database
+    Operations:
+    - OperationMetricsName: GraphLookupUnshardedToUnshardedOneToFew
+      OperationName: RunCommand
+      OperationCommand:
+        aggregate: Collection0
+        pipeline:
+          [
+            {$graphLookup: {
+              from: "Collection1",
+              startWith: "$matchOne",
+              connectFromField: "matchOne",
+              connectToField: "secondMatchOne",
+              as: "matches",
+              maxDepth: 1
+            }},
+            {$project: {a: 1, b: 1, 'matches.a': 1, 'matches.b': 1}}
+          ]
+        cursor: {batchSize: *NumDocs}
+    - OperationMetricsName: GraphLookupUnshardedToUnshardedOneToMany
+      OperationName: RunCommand
+      OperationCommand:
+        aggregate: Collection0
+        pipeline:
+          [
+            {$graphLookup: {
+              from: "Collection1",
+              startWith: "$matchFive",
+              connectFromField: "matchFive",
+              connectToField: "secondMatchFive",
+              as: "matches",
+              maxDepth: 1
+            }},
+            {$project: {a: 1, b: 1, 'matches.a': 1, 'matches.b': 1}}
+          ]
+        cursor: {batchSize: *NumDocs}
+
+AutoRun:
+- When:
+    mongodb_setup:
+      $eq:
+      - replica
+      - shard-lite
+      - shard-lite-all-feature-flags

src/workloads/execution/ShardedLookup.yml → src/workloads/execution/Lookup.yml

@@ -1,13 +1,26 @@
 SchemaVersion: 2018-07-01
 Owner: "@mongodb/query"
 Description: |
-  This test exercises the behavior of $lookup against a sharded foreign collection.
+  This test exercises the behavior of $lookup where either the local or foreign collection is
+  sharded. When both collections are unsharded, the pipeline should be moved to the
+  LookupWithOnlyUnshardedColls workload, so the query can be tested on replica sets as well. We may
+  want to compare results across the two files, so the loading stage (and some queries) should be
+  kept as similar as possible.
+
+  Note: this workload runs only on sharded clusters.
 
   The workload consists of the following phases:
     1. Creating empty sharded collections distributed across all shards in the cluster.
     2. Populating collections with data.
     3. Fsync.
-    4. Running targeted and untargeted $lookups against a sharded foreign collection.
+    4. Running $lookups. This includes but is not limited to cases where the top-level pipelines are
+       targeted and untargeted, the subpipelines are targeted and untargeted, and the subpipelines
+       have cacheable prefixes.
+
+GlobalDefaults:
+  ShardedColl0: &ShardedColl0 Collection0
+  ShardedColl1: &ShardedColl1 Collection1
+  UnshardedColl2: &UnshardedColl2 Collection2
 
 Actors:
 - Name: CreateShardedCollections
@@ -57,6 +70,7 @@ Actors:
     CollectionCount: 3    # Loader will populate 'Collection0', 'Collection1' and 'Collection2'.
     Document:
       shardKey: {^RandomInt: {min: 1, max: 100}}
+      smallInt: {^RandomInt: {min: 1, max: 10}}
       int: {^RandomInt: {min: 1, max: 100}}
       str: {^RandomString: {length: 100}}
   - *Nop
@@ -89,95 +103,168 @@ Actors:
     - OperationMetricsName: TargetedLookupShardedToSharded
       OperationName: RunCommand
       OperationCommand:
-        aggregate: Collection0
+        aggregate: *ShardedColl0
         pipeline:
           [
             {$lookup: {
-              from: "Collection1",
+              from: *ShardedColl1,
               localField: "shardKey",
               foreignField: "shardKey",
               as: "joined"
-            }}
+            }},
+            {$project: {str: 0, "joined.str": 0}}
           ]
         cursor: {batchSize: *NumDocs}
     - OperationMetricsName: UntargetedLookupShardedToSharded
       OperationName: RunCommand
       OperationCommand:
-        aggregate: Collection0
+        aggregate: *ShardedColl0
+        pipeline:
+          [
+            {$lookup: {
+              from: *ShardedColl1,
+              localField: "int",
+              foreignField: "int",
+              as: "joined"
+            }},
+            {$project: {str: 0, "joined.str": 0}}
+          ]
+        cursor: {batchSize: *NumDocs}
+    - OperationMetricsName: UntargetedLookupShardedToUnsharded
+      OperationName: RunCommand
+      OperationCommand:
+        aggregate: *ShardedColl0
         pipeline:
           [
             {$lookup: {
-              from: "Collection1",
+              from: *UnshardedColl2,
               localField: "int",
               foreignField: "int",
               as: "joined"
-            }}
+            }},
+            {$project: {str: 0, "joined.str": 0}}
           ]
         cursor: {batchSize: *NumDocs}
     - OperationMetricsName: UntargetedLookupUnshardedToSharded
       OperationName: RunCommand
       OperationCommand:
-        aggregate: Collection2
+        aggregate: *UnshardedColl2
         pipeline:
           [
             {$lookup: {
-              from: "Collection1",
+              from: *ShardedColl1,
               localField: "int",
               foreignField: "int",
               as: "joined"
-            }}
+            }},
+            {$project: {str: 0, "joined.str": 0}}
           ]
         cursor: {batchSize: *NumDocs}
     - OperationMetricsName: TargetedLookupUnshardedToSharded
       OperationName: RunCommand
       OperationCommand:
-        aggregate: Collection2
+        aggregate: *UnshardedColl2
         pipeline:
           [
             {$lookup: {
-              from: "Collection1",
+              from: *ShardedColl1,
               localField: "shardKey",
               foreignField: "shardKey",
               as: "joined"
-            }}
+            }},
+            {$project: {str: 0, "joined.str": 0}}
           ]
         cursor: {batchSize: *NumDocs}
     - OperationMetricsName: LookupWithCachedPrefixShardedToSharded
       OperationName: RunCommand
       OperationCommand:
-        aggregate: Collection0
+        aggregate: *ShardedColl0
         pipeline:
           [{
             $lookup: {
-              from: Collection1,
+              from: *ShardedColl1,
               let: {localInt: "$int"},
               pipeline: [
                 {$group: {_id: {$sum: ["$int", 1]}}},
                 {$match: {$expr: {$eq: ["$_id", "$$localInt"]}}}
               ],
-              as: matches
+              as: joined
             }
-          }]
+          },
+           {$project: {str: 0, "joined.str": 0}}]
         cursor: {batchSize: *NumDocs}
     - OperationMetricsName: LookupWithCachedPrefixUnshardedToSharded
       OperationName: RunCommand
       OperationCommand:
-        aggregate: Collection2
+        aggregate: *UnshardedColl2
+        pipeline:
+          [{
+            $lookup: {
+              from: *ShardedColl1,
+              let: {localInt: "$int"},
+              pipeline: [
+                {$group: {_id: {$sum: ["$int", 1]}}},
+                {$match: {$expr: {$eq: ["$_id", "$$localInt"]}}}
+              ],
+              as: joined
+            }
+          },
+           {$project: {str: 0, "joined.str": 0}}]
+        cursor: {batchSize: *NumDocs}
+    - OperationMetricsName: LookupWithCachedPrefixShardedToUnsharded
+      OperationName: RunCommand
+      OperationCommand:
+        aggregate: *ShardedColl0
         pipeline:
           [{
             $lookup: {
-              from: Collection1,
+              from: *UnshardedColl2,
               let: {localInt: "$int"},
               pipeline: [
                 {$group: {_id: {$sum: ["$int", 1]}}},
                 {$match: {$expr: {$eq: ["$_id", "$$localInt"]}}}
               ],
-              as: matches
+              as: joined
             }
-          }]
+          },
+           {$project: {str: 0, "joined.str": 0}}]
+        cursor: {batchSize: *NumDocs}
+    - OperationMetricsName: LookupMatchAllShardsShardedToUnsharded
+      OperationName: RunCommand
+      OperationCommand:
+        aggregate: *ShardedColl0
+        pipeline:
+          [
+            {$match: {int: {$lte: 4}}},
+            {$lookup: {
+              from: *UnshardedColl2,
+              let: {smallInt0: "$smallInt"},
+              pipeline: [{$match: {$expr: {$eq: ["$$smallInt0", "$smallInt"]}}}],
+              as: "joined"
+            }},
+            {$project: {str: 0, "joined.str": 0}}
+          ]
+        cursor: {batchSize: *NumDocs}
+    - OperationMetricsName: LookupMatchSomeShardsShardedToUnsharded
+      OperationName: RunCommand
+      OperationCommand:
+        aggregate: *ShardedColl0
+        pipeline:
+          [
+            {$match: {shardKey: {$lte: 4}}},
+            {$lookup: {
+              from: *UnshardedColl2,
+              let: {smallInt0: "$smallInt"},
+              pipeline: [{$match: {$expr: {$eq: ["$$smallInt0", "$smallInt"]}}}],
+              as: "joined"
+            }},
+            {$project: {str: 0, "joined.str": 0}}
+          ]
         cursor: {batchSize: *NumDocs}
 
 AutoRun:
 - When:
     mongodb_setup:
-      $eq: shard-lite-all-feature-flags
+      $eq:
+      - shard-lite
+      - shard-lite-all-feature-flags