Set up VDBE create, edit, delete APIs and connect to dashboard, other…

… UI enhancements (#519)

config/vdbe_demo/imdb_editable_vdbes.json

@@ -0,0 +1,222 @@
+{
+  "schema_name": "imdb_extended_editable_100g",
+  "engines": [
+    {
+      "internal_id": 1,
+      "name": "Ticketing",
+      "max_staleness_ms": 0,
+      "p90_latency_slo_ms": 30,
+      "interface": "postgresql",
+      "tables": [
+        {
+          "name": "theatres",
+          "writable": true
+        },
+        {
+          "name": "showings",
+          "writable": true
+        },
+        {
+          "name": "ticket_orders",
+          "writable": true
+        },
+        {
+          "name": "movie_info",
+          "writable": true
+        },
+        {
+          "name": "aka_title",
+          "writable": true
+        }
+      ],
+      "mapped_to": "aurora"
+    },
+    {
+      "internal_id": 2,
+      "name": "Analytics",
+      "max_staleness_ms": 3600000,
+      "p90_latency_slo_ms": 30000,
+      "interface": "postgresql",
+      "tables": [
+        {
+          "name": "homes",
+          "writable": false
+        },
+        {
+          "name": "theatres",
+          "writable": false
+        },
+        {
+          "name": "showings",
+          "writable": false
+        },
+        {
+          "name": "ticket_orders",
+          "writable": false
+        },
+        {
+          "name": "aka_name",
+          "writable": false
+        },
+        {
+          "name": "aka_title",
+          "writable": false
+        },
+        {
+          "name": "cast_info",
+          "writable": false
+        },
+        {
+          "name": "char_name",
+          "writable": false
+        },
+        {
+          "name": "comp_cast_type",
+          "writable": false
+        },
+        {
+          "name": "company_name",
+          "writable": false
+        },
+        {
+          "name": "company_type",
+          "writable": false
+        },
+        {
+          "name": "complete_cast",
+          "writable": false
+        },
+        {
+          "name": "info_type",
+          "writable": false
+        },
+        {
+          "name": "keyword",
+          "writable": false
+        },
+        {
+          "name": "kind_type",
+          "writable": false
+        },
+        {
+          "name": "link_type",
+          "writable": false
+        },
+        {
+          "name": "movie_companies",
+          "writable": false
+        },
+        {
+          "name": "movie_info_idx",
+          "writable": false
+        },
+        {
+          "name": "movie_keyword",
+          "writable": false
+        },
+        {
+          "name": "movie_link",
+          "writable": false
+        },
+        {
+          "name": "name",
+          "writable": false
+        },
+        {
+          "name": "role_type",
+          "writable": false
+        },
+        {
+          "name": "title",
+          "writable": false
+        },
+        {
+          "name": "movie_info",
+          "writable": false
+        },
+        {
+          "name": "person_info",
+          "writable": false
+        }
+      ],
+      "mapped_to": "redshift"
+    }
+  ],
+  "tables": [
+    {
+      "name": "homes"
+    },
+    {
+      "name": "theatres"
+    },
+    {
+      "name": "showings"
+    },
+    {
+      "name": "ticket_orders"
+    },
+    {
+      "name": "aka_name"
+    },
+    {
+      "name": "aka_title"
+    },
+    {
+      "name": "cast_info"
+    },
+    {
+      "name": "char_name"
+    },
+    {
+      "name": "comp_cast_type"
+    },
+    {
+      "name": "company_name"
+    },
+    {
+      "name": "company_type"
+    },
+    {
+      "name": "complete_cast"
+    },
+    {
+      "name": "info_type"
+    },
+    {
+      "name": "keyword"
+    },
+    {
+      "name": "kind_type"
+    },
+    {
+      "name": "link_type"
+    },
+    {
+      "name": "movie_companies"
+    },
+    {
+      "name": "movie_info_idx"
+    },
+    {
+      "name": "movie_keyword"
+    },
+    {
+      "name": "movie_link"
+    },
+    {
+      "name": "name"
+    },
+    {
+      "name": "role_type"
+    },
+    {
+      "name": "title"
+    },
+    {
+      "name": "movie_info"
+    },
+    {
+      "name": "person_info"
+    }
+  ]
+}

config/vdbe_demo/imdb_extended_vdbes.json

@@ -1,8 +1,9 @@
 {
-  "schema_name": "imdb_extended",
+  "schema_name": "imdb_extended_100g",
   "engines": [
     {
-      "name": "VDBE (T)",
+      "internal_id": 1,
+      "name": "Ticketing",
       "max_staleness_ms": 0,
       "p90_latency_slo_ms": 30,
       "interface": "postgresql",
@@ -31,7 +32,8 @@
       "mapped_to": "aurora"
     },
     {
-      "name": "VDBE (A)",
+      "internal_id": 2,
+      "name": "Analytics",
       "max_staleness_ms": 3600000,
       "p90_latency_slo_ms": 30000,
       "interface": "postgresql",
@@ -137,7 +139,7 @@
           "writable": false
         }
       ],
-      "mapped_to": "redshift"
+      "mapped_to": "aurora"
     }
   ],
   "tables": [

src/brad/admin/modify_blueprint.py

@@ -22,6 +22,7 @@
 from brad.routing.policy import RoutingPolicy
 from brad.routing.tree_based.forest_policy import ForestPolicy
 from brad.routing.rule_based import RuleBased
+from brad.vdbe.models import VirtualInfrastructure
 
 logger = logging.getLogger(__name__)
 
@@ -102,6 +103,11 @@ def register_admin_action(subparser) -> None:
         "on the specified engines. Overridden by --place-tables-everywhere. Format "
         "argument as a string of the form: table1=engine1,engine2;table2=engine3;",
     )
+    parser.add_argument(
+        "--place-vdbe-tables",
+        type=str,
+        help="Places tables on the specified engines in the given VDBE JSON file.",
+    )
     parser.add_argument(
         "--set-routing-policy",
         choices=[
@@ -323,6 +329,18 @@ def modify_blueprint(args) -> None:
             new_placement[tbl] = Engine.from_bitmap(Engine.bitmap_all())
         enum_blueprint.set_table_locations(new_placement)
 
+    if args.place_vdbe_tables is not None:
+        new_placement = {}
+        with open(args.place_vdbe_tables, "r", encoding="utf-8") as f:
+            vinfra = VirtualInfrastructure.model_validate_json(f.read())
+        for engine in vinfra.engines:
+            for table in engine.tables:
+                try:
+                    new_placement[table.name].append(engine.mapped_to)
+                except KeyError:
+                    new_placement[table.name] = [engine.mapped_to]
+        enum_blueprint.set_table_locations(new_placement)
+
     # 5. Modify routing policy as needed.
     if args.set_routing_policy is not None:
         if args.set_routing_policy == "always_redshift":

src/brad/daemon/daemon.py

@@ -132,7 +132,8 @@ def __init__(
         load_vdbe_path = self._config.bootstrap_vdbe_path()
         if load_vdbe_path is not None:
             self._vdbe_manager: Optional[VdbeManager] = VdbeManager.load_from(
-                load_vdbe_path
+                load_vdbe_path,
+                starting_port=9876,
             )
         else:
             self._vdbe_manager = None