feat: reuse worker threads for bru file parsing (#4054)

usebruno · Feb 18, 2025 · 31409c6 · 31409c6
1 parent dfb0b1b
commit 31409c6
Show file tree

Hide file tree

Showing 9 changed files with 104 additions and 39 deletions.
diff --git a/packages/bruno-app/src/components/CollectionSettings/Overview/Info/index.js b/packages/bruno-app/src/components/CollectionSettings/Overview/Info/index.js
@@ -1,10 +1,14 @@
-import React from 'react';
+import React from "react";
 import { getTotalRequestCountInCollection } from 'utils/collections/';
-import { IconFolder, IconFileOff, IconWorld, IconApi } from '@tabler/icons';
+import { IconFolder, IconWorld, IconApi, IconClock } from '@tabler/icons';
+import { areItemsLoading, getItemsLoadStats } from "utils/collections/index";
 
 const Info = ({ collection }) => {
   const totalRequestsInCollection = getTotalRequestCountInCollection(collection);
 
+  const isCollectionLoading = areItemsLoading(collection);
+  const { loading: itemsLoadingCount, total: totalItems } = getItemsLoadStats(collection);
+
   return (
     <div className="w-full flex flex-col h-fit">
       <div className="rounded-lg py-6">
@@ -42,8 +46,10 @@ const Info = ({ collection }) => {
             </div>
             <div className="ml-4">
               <div className="font-semibold text-sm">Requests</div>
-              <div className="mt-1 text-sm text-muted">
-                {totalRequestsInCollection} request{totalRequestsInCollection !== 1 ? 's' : ''} in collection
+              <div className="mt-1 text-sm text-muted font-mono">
+                {
+                  isCollectionLoading? `${totalItems - itemsLoadingCount} out of ${totalItems} requests in the collection loaded` : `${totalRequestsInCollection} request${totalRequestsInCollection !== 1 ? 's' : ''} in collection`
+                }
               </div>
             </div>
           </div>
@@ -53,4 +59,4 @@ const Info = ({ collection }) => {
   );
 };
 
-export default Info;
+export default Info;
diff --git a/packages/bruno-app/src/components/CollectionSettings/Overview/RequestsNotLoaded/index.js b/packages/bruno-app/src/components/CollectionSettings/Overview/RequestsNotLoaded/index.js
@@ -2,15 +2,45 @@ import React from 'react';
 import { flattenItems } from "utils/collections";
 import { IconAlertTriangle } from '@tabler/icons';
 import StyledWrapper from "./StyledWrapper";
+import { useDispatch, useSelector } from 'react-redux';
+import { isItemARequest, itemIsOpenedInTabs } from 'utils/tabs/index';
+import { getDefaultRequestPaneTab } from 'utils/collections/index';
+import { addTab, focusTab } from 'providers/ReduxStore/slices/tabs';
+import { hideHomePage } from 'providers/ReduxStore/slices/app';
 
 const RequestsNotLoaded = ({ collection }) => {
+  const dispatch = useDispatch();
+  const tabs = useSelector((state) => state.tabs.tabs);
   const flattenedItems = flattenItems(collection.items);
   const itemsFailedLoading = flattenedItems?.filter(item => item?.partial && !item?.loading);
 
   if (!itemsFailedLoading?.length) {
     return null;
   }
 
+  const handleRequestClick = (item) => e => {
+    e.preventDefault();
+    if (isItemARequest(item)) {
+      dispatch(hideHomePage());
+      if (itemIsOpenedInTabs(item, tabs)) {
+        dispatch(
+          focusTab({
+            uid: item.uid
+          })
+        );
+        return;
+      }
+      dispatch(
+        addTab({
+          uid: item.uid,
+          collectionUid: collection.uid,
+          requestPaneTab: getDefaultRequestPaneTab(item)
+        })
+      );
+      return;
+    }
+  }
+
   return (
     <StyledWrapper className="w-full card my-2">
       <div className="flex items-center gap-2 px-3 py-2 title bg-yellow-50 dark:bg-yellow-900/20">
@@ -31,7 +61,7 @@ const RequestsNotLoaded = ({ collection }) => {
         <tbody>
           {flattenedItems?.map((item, index) => (
             item?.partial && !item?.loading ? (
-              <tr key={index}>
+              <tr key={index} className='cursor-pointer' onClick={handleRequestClick(item)}>
                 <td className="py-1.5 px-3">
                   {item?.pathname?.split(`${collection?.pathname}/`)?.[1]}
                 </td>

diff --git a/packages/bruno-app/src/components/RequestTabPanel/RequestNotLoaded/index.js b/packages/bruno-app/src/components/RequestTabPanel/RequestNotLoaded/index.js
@@ -21,18 +21,18 @@ const RequestNotLoaded = ({ collection, item }) => {
             <IconFile size={16} strokeWidth={1.5} className="text-gray-400" />
             File Info
           </div>
-          <div className='hr'/>
+          <div className='hr' />
 
           <div className='flex items-center mt-2'>
             <span className='w-12 mr-2 text-muted'>Name:</span>
             <div>{item?.name}</div>
           </div>
-          
+
           <div className='flex items-center mt-1'>
             <span className='w-12 mr-2 text-muted'>Path:</span>
             <div className='break-all'>{item?.pathname}</div>
           </div>
-          
+
           <div className='flex items-center mt-1 pb-4'>
             <span className='w-12 mr-2 text-muted'>Size:</span>
             <div>{item?.size?.toFixed?.(2)} MB</div>
@@ -67,7 +67,7 @@ const RequestNotLoaded = ({ collection, item }) => {
 
           {item?.loading && (
             <>
-              <div className='hr mt-4'/>
+              <div className='hr mt-4' />
               <div className='flex items-center gap-2 mt-4'>
                 <IconLoader2 className="animate-spin" size={16} strokeWidth={2} />
                 <span>Loading...</span>

diff --git a/packages/bruno-app/src/providers/ReduxStore/slices/collections/index.js b/packages/bruno-app/src/providers/ReduxStore/slices/collections/index.js
@@ -59,7 +59,9 @@ export const collectionsSlice = createSlice({
     updateCollectionMountStatus: (state, action) => {
       const collection = findCollectionByUid(state.collections, action.payload.collectionUid);
       if (collection) {
-        collection.mountStatus = action.payload.mountStatus;
+        if (action.payload.mountStatus) {
+          collection.mountStatus = action.payload.mountStatus;
+        }
       }
     },
     setCollectionSecurityConfig: (state, action) => {

diff --git a/packages/bruno-app/src/utils/collections/index.js b/packages/bruno-app/src/utils/collections/index.js
@@ -137,6 +137,20 @@ export const areItemsLoading = (folder) => {
   }, false);
 }
 
+export const getItemsLoadStats = (folder) => {
+  let loadingCount = 0;
+  let flattenedItems = flattenItems(folder.items);
+  flattenedItems?.forEach(i => {
+    if(i?.loading) {
+      loadingCount += 1;
+    }
+  });
+  return {
+    loading: loadingCount,
+    total: flattenedItems?.length
+  };
+}
+
 export const moveCollectionItem = (collection, draggedItem, targetItem) => {
   let draggedItemParent = findParentItemInCollection(collection, draggedItem.uid);
 

diff --git a/packages/bruno-electron/src/bru/workers/scripts/bru-to-json.js b/packages/bruno-electron/src/bru/workers/scripts/bru-to-json.js
@@ -1,14 +1,16 @@
-const { workerData, parentPort } = require('worker_threads');
+const { parentPort } = require('worker_threads');
 const {
   bruToJsonV2,
 } = require('@usebruno/lang');
 
-try {
-  const bru = workerData;
-  const json = bruToJsonV2(bru);
-  parentPort.postMessage(json);
-}
-catch(error) {
-  console.error(error);
-  parentPort.postMessage({ error: error?.message });
-}
+parentPort.on('message', (workerData) => {
+  try {
+    const bru = workerData;
+    const json = bruToJsonV2(bru);
+    parentPort.postMessage(json);
+  }
+  catch(error) {
+    console.error(error);
+    parentPort.postMessage({ error: error?.message });
+  }
+});
diff --git a/packages/bruno-electron/src/bru/workers/scripts/json-to-bru.js b/packages/bruno-electron/src/bru/workers/scripts/json-to-bru.js
@@ -1,13 +1,16 @@
-const { workerData, parentPort } = require('worker_threads');
+const { parentPort } = require('worker_threads');
 const {
   jsonToBruV2,
 } = require('@usebruno/lang');
-try {
-  const json = workerData;
-  const bru = jsonToBruV2(json);
-  parentPort.postMessage(bru);
-}
-catch(error) {
-  console.error(error);
-  parentPort.postMessage({ error: error?.message });
-}
+
+parentPort.on('message', (workerData) => {
+  try {
+    const json = workerData;
+    const bru = jsonToBruV2(json);
+    parentPort.postMessage(bru);
+  }
+  catch(error) {
+    console.error(error);
+    parentPort.postMessage({ error: error?.message });
+  }
+});
diff --git a/packages/bruno-electron/src/ipc/collection.js b/packages/bruno-electron/src/ipc/collection.js
@@ -40,9 +40,9 @@ const collectionSecurityStore = new CollectionSecurityStore();
 const uiStateSnapshotStore = new UiStateSnapshotStore();
 
 // size and file count limits to determine whether the bru files in the collection should be loaded asynchronously or not.
-const MAX_COLLECTION_SIZE_IN_MB = 5;
-const MAX_SINGLE_FILE_SIZE_IN_COLLECTION_IN_MB = 2;
-const MAX_COLLECTION_FILES_COUNT = 100;
+const MAX_COLLECTION_SIZE_IN_MB = 20;
+const MAX_SINGLE_FILE_SIZE_IN_COLLECTION_IN_MB = 5;
+const MAX_COLLECTION_FILES_COUNT = 2000;
 
 const envHasSecrets = (environment = {}) => {
   const secrets = _.filter(environment.variables, (v) => v.secret);

diff --git a/packages/bruno-electron/src/workers/index.js b/packages/bruno-electron/src/workers/index.js
@@ -4,8 +4,18 @@ class WorkerQueue {
   constructor() {
     this.queue = [];
     this.isProcessing = false;
+    this.workers = {};
   }
 
+  async getWorkerForScriptPath(scriptPath) {
+    if (!this.workers) this.workers = {}; 
+    let worker = this.workers[scriptPath];
+    if (!worker || worker.threadId === -1) {
+      this.workers[scriptPath] = worker = new Worker(scriptPath);
+    }
+    return worker;
+  }
+
   async enqueue(task) {
     const { priority, scriptPath, data } = task;
 
@@ -36,22 +46,20 @@ class WorkerQueue {
   }
 
   async runWorker({ scriptPath, data }) {
-    return new Promise((resolve, reject) => {
-      const worker = new Worker(scriptPath, { workerData: data });
+    return new Promise(async (resolve, reject) => {
+      let worker = await this.getWorkerForScriptPath(scriptPath);
+      worker.postMessage(data);
       worker.on('message', (data) => {
         if (data?.error) {
           reject(new Error(data?.error));
         }
         resolve(data);
-        worker.terminate();
       });
       worker.on('error', (error) => {
         reject(error);
-        worker.terminate();
       });
       worker.on('exit', (code) => {
         reject(new Error(`stopped with  ${code} exit code`));
-        worker.terminate();
       });
     });
   }