WIP: feat: add volume serving plugin

apps/dev-playground/client/src/appKitTypes.d.ts

@@ -3,7 +3,7 @@ import "@databricks/app-kit-ui/react";
 
 declare module "@databricks/app-kit-ui/react" {
   interface PluginRegistry {
-    reconnect: {
+    "reconnect": {
       "/": {
           message: string;
       };
@@ -15,7 +15,7 @@ declare module "@databricks/app-kit-ui/react" {
           content: string;
       };
     }
-    analytics: {
+    "analytics": {
       "/users/me/query/:query_key": {
           chunk_index: number;
           row_offset: number;
@@ -32,6 +32,7 @@ declare module "@databricks/app-kit-ui/react" {
   }
 
   interface QueryRegistry {
+
     apps_list: {
         id: string;
         name: string;
@@ -61,5 +62,4 @@ declare module "@databricks/app-kit-ui/react" {
         total_cost_usd: number;
     }[];
   }
-
 }

apps/dev-playground/client/src/hooks/use-directory-listing.ts

@@ -0,0 +1,190 @@
+import { useState, useEffect, useCallback } from "react";
+
+interface FileItem {
+  name: string;
+  path: string;
+  isDirectory: boolean;
+  size?: number;
+  mimeType?: string | null;
+}
+
+interface DirectoryListing {
+  path: string;
+  files: FileItem[];
+}
+
+export function useDirectoryListing(
+  initialPath: string = "/",
+  onPathChange?: (path: string) => void,
+  batchSize: number = 50,
+) {
+  const [currentPath, setCurrentPath] = useState(initialPath);
+  const [directoryListing, setDirectoryListing] =
+    useState<DirectoryListing | null>(null);
+  const [loading, setLoading] = useState(false);
+  const [error, setError] = useState<string | null>(null);
+
+  const fetchPath = useCallback(
+    async (path: string) => {
+      setLoading(true);
+      setError(null);
+      setDirectoryListing(null);
+
+      // Batch-related variables accessible to both try and catch blocks
+      const files: FileItem[] = [];
+      let dirPath = path;
+      let batchBuffer: FileItem[] = [];
+      const effectiveBatchSize = Math.max(1, batchSize); // Clamp to minimum 1
+
+      // Flush function - centralized batch update logic
+      const flushBatch = () => {
+        if (batchBuffer.length > 0) {
+          files.push(...batchBuffer);
+          setDirectoryListing({ path: dirPath, files: [...files] });
+          batchBuffer = [];
+        }
+      };
+
+      try {
+        // Stream directory listing from NDJSON response
+        const response = await fetch(`/api/volume-serving${path}`);
+
+        if (!response.ok) {
+          throw new Error(`HTTP error! status: ${response.status}`);
+        }
+
+        // Read the stream line by line
+        const reader = response.body?.getReader();
+        const decoder = new TextDecoder();
+        let buffer = "";
+
+        if (reader) {
+          while (true) {
+            const { done, value } = await reader.read();
+
+            if (done) {
+              flushBatch(); // Flush remaining files
+              break;
+            }
+
+            buffer += decoder.decode(value, { stream: true });
+            const lines = buffer.split("\n");
+
+            // Process all complete lines
+            for (let i = 0; i < lines.length - 1; i++) {
+              const line = lines[i].trim();
+              if (line) {
+                const data = JSON.parse(line);
+
+                if (data.type === "metadata") {
+                  dirPath = data.path;
+                } else if (data.type === "file") {
+                  const fileItem: FileItem = {
+                    name: data.name,
+                    path: data.path,
+                    isDirectory: data.isDirectory,
+                    size: data.size,
+                    mimeType: data.mimeType,
+                  };
+
+                  batchBuffer.push(fileItem);
+
+                  // Flush when batch is full
+                  if (batchBuffer.length >= effectiveBatchSize) {
+                    flushBatch();
+                  }
+                }
+              }
+            }
+
+            // Keep the last incomplete line in the buffer
+            buffer = lines[lines.length - 1];
+          }
+        }
+      } catch (err) {
+        flushBatch(); // Flush partial batch before error state
+        setError(err instanceof Error ? err.message : "Failed to fetch path");
+      } finally {
+        setLoading(false);
+      }
+    },
+    [batchSize],
+  );
+
+  const navigateUp = () => {
+    // Remove trailing slash if present
+    const cleanPath =
+      currentPath.endsWith("/") && currentPath !== "/"
+        ? currentPath.slice(0, -1)
+        : currentPath;
+
+    // Get parent directory
+    const parentPath =
+      cleanPath.substring(0, cleanPath.lastIndexOf("/")) || "/";
+    const normalizedParent = parentPath === "/" ? "/" : `${parentPath}/`;
+
+    setCurrentPath(normalizedParent);
+    onPathChange?.(normalizedParent);
+    fetchPath(normalizedParent);
+  };
+
+  const handleNavigate = (file: FileItem) => {
+    // Handle ".." navigation
+    if (file.name === "..") {
+      navigateUp();
+      return;
+    }
+
+    if (file.isDirectory) {
+      // Navigate to directory
+      const newPath = file.path;
+      setCurrentPath(newPath);
+      onPathChange?.(newPath);
+      fetchPath(newPath);
+    } else {
+      // Open file in new window
+      window.open(`/api/volume-serving${file.path}`, "_blank");
+    }
+  };
+
+  // Load directory when initialPath changes (including from URL)
+  useEffect(() => {
+    setCurrentPath(initialPath);
+    fetchPath(initialPath);
+  }, [initialPath, fetchPath]);
+
+  // Prepare files list with ".." entry if not in root, sorted with folders first
+  const filesWithNavigation = directoryListing
+    ? [
+        ...(currentPath !== "/"
+          ? [
+              {
+                name: "..",
+                path: "",
+                isDirectory: true,
+                size: undefined,
+                mimeType: null,
+              },
+            ]
+          : []),
+        // Sort: directories first, then by name
+        ...directoryListing.files.sort((a, b) => {
+          if (a.isDirectory === b.isDirectory) {
+            return a.name.localeCompare(b.name);
+          }
+          return a.isDirectory ? -1 : 1;
+        }),
+      ]
+    : [];
+
+  return {
+    currentPath,
+    directoryListing,
+    loading,
+    error,
+    filesWithNavigation,
+    fetchPath,
+    navigateUp,
+    handleNavigate,
+  };
+}

apps/dev-playground/client/src/routeTree.gen.ts

@@ -9,12 +9,18 @@
 // Additionally, you should also exclude this file from your linter and/or formatter to prevent it from being checked or modified.
 
 import { Route as rootRouteImport } from "./routes/__root";
+import { Route as VolumeServingRouteRouteImport } from "./routes/volume-serving.route";
 import { Route as TelemetryRouteRouteImport } from "./routes/telemetry.route";
 import { Route as ReconnectRouteRouteImport } from "./routes/reconnect.route";
 import { Route as DataVisualizationRouteRouteImport } from "./routes/data-visualization.route";
 import { Route as AnalyticsRouteRouteImport } from "./routes/analytics.route";
 import { Route as IndexRouteImport } from "./routes/index";
 
+const VolumeServingRouteRoute = VolumeServingRouteRouteImport.update({
+  id: "/volume-serving",
+  path: "/volume-serving",
+  getParentRoute: () => rootRouteImport,
+} as any);
 const TelemetryRouteRoute = TelemetryRouteRouteImport.update({
   id: "/telemetry",
   path: "/telemetry",
@@ -47,13 +53,15 @@ export interface FileRoutesByFullPath {
   "/data-visualization": typeof DataVisualizationRouteRoute;
   "/reconnect": typeof ReconnectRouteRoute;
   "/telemetry": typeof TelemetryRouteRoute;
+  "/volume-serving": typeof VolumeServingRouteRoute;
 }
 export interface FileRoutesByTo {
   "/": typeof IndexRoute;
   "/analytics": typeof AnalyticsRouteRoute;
   "/data-visualization": typeof DataVisualizationRouteRoute;
   "/reconnect": typeof ReconnectRouteRoute;
   "/telemetry": typeof TelemetryRouteRoute;
+  "/volume-serving": typeof VolumeServingRouteRoute;
 }
 export interface FileRoutesById {
   __root__: typeof rootRouteImport;
@@ -62,6 +70,7 @@ export interface FileRoutesById {
   "/data-visualization": typeof DataVisualizationRouteRoute;
   "/reconnect": typeof ReconnectRouteRoute;
   "/telemetry": typeof TelemetryRouteRoute;
+  "/volume-serving": typeof VolumeServingRouteRoute;
 }
 export interface FileRouteTypes {
   fileRoutesByFullPath: FileRoutesByFullPath;
@@ -70,16 +79,24 @@ export interface FileRouteTypes {
     | "/analytics"
     | "/data-visualization"
     | "/reconnect"
-    | "/telemetry";
+    | "/telemetry"
+    | "/volume-serving";
   fileRoutesByTo: FileRoutesByTo;
-  to: "/" | "/analytics" | "/data-visualization" | "/reconnect" | "/telemetry";
+  to:
+    | "/"
+    | "/analytics"
+    | "/data-visualization"
+    | "/reconnect"
+    | "/telemetry"
+    | "/volume-serving";
   id:
     | "__root__"
     | "/"
     | "/analytics"
     | "/data-visualization"
     | "/reconnect"
-    | "/telemetry";
+    | "/telemetry"
+    | "/volume-serving";
   fileRoutesById: FileRoutesById;
 }
 export interface RootRouteChildren {
@@ -88,10 +105,18 @@ export interface RootRouteChildren {
   DataVisualizationRouteRoute: typeof DataVisualizationRouteRoute;
   ReconnectRouteRoute: typeof ReconnectRouteRoute;
   TelemetryRouteRoute: typeof TelemetryRouteRoute;
+  VolumeServingRouteRoute: typeof VolumeServingRouteRoute;
 }
 
 declare module "@tanstack/react-router" {
   interface FileRoutesByPath {
+    "/volume-serving": {
+      id: "/volume-serving";
+      path: "/volume-serving";
+      fullPath: "/volume-serving";
+      preLoaderRoute: typeof VolumeServingRouteRouteImport;
+      parentRoute: typeof rootRouteImport;
+    };
     "/telemetry": {
       id: "/telemetry";
       path: "/telemetry";
@@ -136,6 +161,7 @@ const rootRouteChildren: RootRouteChildren = {
   DataVisualizationRouteRoute: DataVisualizationRouteRoute,
   ReconnectRouteRoute: ReconnectRouteRoute,
   TelemetryRouteRoute: TelemetryRouteRoute,
+  VolumeServingRouteRoute: VolumeServingRouteRoute,
 };
 export const routeTree = rootRouteImport
   ._addFileChildren(rootRouteChildren)

apps/dev-playground/client/src/routes/__root.tsx

@@ -56,6 +56,14 @@ function RootComponent() {
                     Telemetry
                   </Button>
                 </Link>
+                <Link to="/volume-serving" className="no-underline">
+                  <Button
+                    variant="ghost"
+                    className="text-gray-700 hover:text-gray-900"
+                  >
+                    Volume Serving
+                  </Button>
+                </Link>
               </div>
             </nav>
           </div>

apps/dev-playground/client/src/routes/index.tsx

@@ -103,6 +103,24 @@ function IndexRoute() {
               </Button>
             </div>
           </Card>
+
+          <Card className="p-6 hover:shadow-lg transition-shadow cursor-pointer">
+            <div className="flex flex-col h-full">
+              <h3 className="text-2xl font-semibold text-gray-900 mb-3">
+                Volume Serving
+              </h3>
+              <p className="text-gray-600 mb-6 flex-grow">
+                Serve files and data directly from Databricks volumes with high
+                performance and scalability.
+              </p>
+              <Button
+                onClick={() => navigate({ to: "/volume-serving" })}
+                className="w-full"
+              >
+                Explore Volume Serving
+              </Button>
+            </div>
+          </Card>
         </div>
 
         <div className="text-center pt-12 border-t border-gray-200">

apps/dev-playground/client/src/routes/telemetry.route.tsx

@@ -1,8 +1,8 @@
 import { createFileRoute, retainSearchParams } from "@tanstack/react-router";
+import { Activity, Loader2 } from "lucide-react";
+import { useState } from "react";
 import { Button } from "@/components/ui/button";
 import { Card } from "@/components/ui/card";
-import { useState } from "react";
-import { Activity, Loader2 } from "lucide-react";
 
 export const Route = createFileRoute("/telemetry")({
   component: TelemetryRoute,