Virtualized timeline (#38)

* Virtual list for feed

* cleanup

* Cleanup

* Try react-virtual

* Final implementation

.erb/configs/webpack.config.renderer.dev.ts

@@ -163,8 +163,6 @@ const configuration: webpack.Configuration = {
       isDevelopment: process.env.NODE_ENV !== 'production',
       nodeModules: webpackPaths.appNodeModulesPath,
     }),
-
-    million.webpack({ auto: true, mute: true }),
   ],
 
   node: {

.erb/configs/webpack.config.renderer.prod.ts

@@ -136,8 +136,6 @@ const configuration: webpack.Configuration = {
     new webpack.DefinePlugin({
       'process.type': '"renderer"',
     }),
-
-    million.webpack({ auto: true, mute: true }),
   ],
 };
 

package.json

@@ -32,7 +32,7 @@
     "build:renderer": "cross-env NODE_ENV=production TS_NODE_TRANSPILE_ONLY=true webpack --config ./.erb/configs/webpack.config.renderer.prod.ts",
     "postinstall": "ts-node .erb/scripts/check-native-dep.js && electron-builder install-app-deps && cross-env NODE_ENV=development TS_NODE_TRANSPILE_ONLY=true webpack --config ./.erb/configs/webpack.config.renderer.dev.dll.ts",
     "lint": "cross-env NODE_ENV=development eslint . --ext .js,.jsx,.ts,.tsx",
-    "package": "ts-node ./.erb/scripts/clean.js dist && npm run build && electron-builder build --mac --publish never",
+    "package": "ts-node ./.erb/scripts/clean.js dist && npm run build && electron-builder build --mac --publish always",
     "rebuild": "electron-rebuild --parallel --types prod,dev,optional --module-dir release/app",
     "start": "ts-node ./.erb/scripts/check-port-in-use.js && npm run start:renderer",
     "start:main": "cross-env NODE_ENV=development electronmon -r ts-node/register/transpile-only .",
@@ -91,6 +91,7 @@
     "@radix-ui/react-alert-dialog": "^1.0.4",
     "@radix-ui/react-dialog": "^1.0.4",
     "@radix-ui/react-dropdown-menu": "^2.0.5",
+    "@tanstack/react-virtual": "^3.0.1",
     "@tiptap/extension-character-count": "^2.1.12",
     "@tiptap/extension-link": "^2.1.12",
     "@tiptap/extension-placeholder": "^2.0.3",
@@ -113,10 +114,11 @@
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-router-dom": "^6.11.2",
-    "react-textarea-autosize": "^8.5.3"
+    "react-textarea-autosize": "^8.5.3",
+    "react-virtuoso": "^4.6.2"
   },
   "devDependencies": {
-    "@adobe/css-tools": "^4.3.1",
+    "@adobe/css-tools": "^4.3.2",
     "@babel/traverse": "^7.23.2",
     "@electron/notarize": "^1.2.3",
     "@electron/rebuild": "^3.2.13",

release/app/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pile",
-  "version": "0.8.2",
+  "version": "0.9.0",
   "description": "Pile: Everyday journal and thought companion.",
   "license": "MIT",
   "author": {

src/main/handlers/index.ts

@@ -17,6 +17,11 @@ ipcMain.handle('index-add', (event, filePath) => {
   return index;
 });
 
+ipcMain.handle('index-update', (event, filePath, data) => {
+  const index = pileIndex.update(filePath, data);
+  return index;
+});
+
 ipcMain.handle('index-remove', (event, filePath) => {
   const index = pileIndex.remove(filePath);
   return index;

src/main/utils/pileVectorIndex.js

@@ -4,6 +4,7 @@ const glob = require('glob');
 const matter = require('gray-matter');
 const keytar = require('keytar');
 const pileIndex = require('./pileIndex');
+const { BrowserWindow } = require('electron');
 
 const {
   TextNode,
@@ -46,6 +47,13 @@ class PileVectorIndex {
     await this.initChatEngine();
   }
 
+  async sendMessageToRenderer(channel = 'status', message) {
+    let win = BrowserWindow.getFocusedWindow();
+    if (win && !win.isDestroyed()) {
+      win.webContents.send(channel, message);
+    }
+  }
+
   async setAPIKeyToEnv() {
     const apikey = await keytar.getPassword('pile', 'aikey');
     if (!apikey) {
@@ -95,7 +103,7 @@ class PileVectorIndex {
       );
 
       // Build the vector store for existing entries.
-      console.log('🛠️ Building vector index');
+      console.log('🛠️ Building fresh vector index');
       this.rebuildVectorIndex(this.pilePath);
     }
   }
@@ -267,6 +275,8 @@ class PileVectorIndex {
     const index = new Map(JSON.parse(data));
     const documents = [];
 
+    let count = 1;
+
     console.log('🟢 Rebuilding vector store...');
     // it makes sense to compile each thread into one document before
     // injesting it here... the metadata can includes the relative paths of
@@ -315,13 +325,36 @@ class PileVectorIndex {
 
           const replies = await Promise.all(childNodePromises);
           thread.relationships[NodeRelationship.CHILD] = replies;
+
           this.addDocument(thread);
+
+          this.sendMessageToRenderer('vector-index', {
+            type: 'indexing',
+            count: count,
+            total: index.size,
+            message: `Indexed entry ${count}/${index.size}`,
+          });
+
           console.log('✅ Successfully indexed file', relativeFilePath);
         }
+        count++;
       } catch (error) {
+        this.sendMessageToRenderer('vector-index', {
+          type: 'indexing',
+          count: count,
+          total: index.size,
+          message: `Failed to index an entry. Skipping ${count}/${index.size}`,
+        });
         console.log('❌ Failed to embed and index:', relativeFilePath);
       }
     }
+
+    this.sendMessageToRenderer('vector-index', {
+      type: 'done',
+      count: index.size,
+      total: index.size,
+      message: `Indexing complete`,
+    });
     console.log('🟢 Finished building index');
   }
 

src/renderer/context/IndexContext.js

@@ -54,6 +54,18 @@ export const IndexContextProvider = ({ children }) => {
     [currentPile]
   );
 
+  const updateIndex = useCallback(async (filePath, data) => {
+    // setIndex((prevMap) => {
+    //   const newMap = new Map(prevMap);
+    //   newMap.set(filePath, data);
+    //   return newMap;
+    // });
+
+    window.electron.ipc.invoke('index-update', filePath, data).then((index) => {
+      // setIndex(index);
+    });
+  }, []);
+
   const removeIndex = useCallback(async (filePath) => {
     window.electron.ipc.invoke('index-remove', filePath).then((index) => {
       setIndex(index);
@@ -92,6 +104,7 @@ export const IndexContextProvider = ({ children }) => {
     initVectorIndex,
     rebuildVectorIndex,
     getVectorIndex,
+    updateIndex,
     query,
   };
 

src/renderer/context/TimelineContext.js

@@ -4,20 +4,40 @@ import {
   useContext,
   useEffect,
   useCallback,
+  useRef,
 } from 'react';
 import { useLocation } from 'react-router-dom';
 import debounce from 'renderer/utils/debounce';
 
 export const TimelineContext = createContext();
 
 export const TimelineContextProvider = ({ children }) => {
-  const [closestDate, _setClosestDate] = useState(new Date());
+  const [visibleIndex, _setVisibleIndex] = useState(0);
+  const [closestDate, setClosestDate] = useState(new Date());
+  const virtualListRef = useRef(null);
 
-  const setClosestDate = debounce((val) => {
-    _setClosestDate(val);
+  const setVisibleIndex = debounce((index) => {
+    _setVisibleIndex(index);
   }, 15);
 
-  const timelineContextValue = { closestDate, setClosestDate };
+  const scrollToIndex = useCallback((index = 0) => {
+    if (!virtualListRef.current) return;
+    if (index == -1) return;
+    virtualListRef.current.scrollToIndex({
+      index,
+      align: 'end',
+      behavior: 'auto',
+    });
+  }, []);
+
+  const timelineContextValue = {
+    virtualListRef,
+    visibleIndex,
+    closestDate,
+    setClosestDate,
+    scrollToIndex,
+    setVisibleIndex,
+  };
 
   return (
     <TimelineContext.Provider value={timelineContextValue}>

src/renderer/hooks/useIPCListener.js

@@ -4,7 +4,7 @@ const useIPCListener = (channel, initialData) => {
   const [data, setData] = useState(initialData);
 
   useEffect(() => {
-    const handler = (event, newData) => {
+    const handler = (newData) => {
       setData(newData);
     };
 

src/renderer/hooks/useWindowResize.js

@@ -0,0 +1,18 @@
+import { useState, useLayoutEffect } from 'react';
+
+export const useWindowResize = () => {
+  const [size, setSize] = useState([0, 0]);
+
+  useLayoutEffect(() => {
+    function updateSize() {
+      setSize([window.innerWidth, window.innerHeight]);
+    }
+
+    window.addEventListener('resize', updateSize);
+    updateSize();
+
+    return () => window.removeEventListener('resize', updateSize);
+  }, []);
+
+  return size;
+};

src/renderer/pages/Pile/Editor/Attachments/index.jsx

@@ -4,11 +4,11 @@ import { DiscIcon, PhotoIcon, TrashIcon, TagIcon } from 'renderer/icons';
 import { motion } from 'framer-motion';
 import { usePilesContext } from 'renderer/context/PilesContext';
 
-export default function Attachments({
+const Attachments = ({
   post,
   onRemoveAttachment = () => {},
   editable = false,
-}) {
+}) => {
   const { getCurrentPilePath } = usePilesContext();
 
   if (!post) return;
@@ -42,4 +42,6 @@ export default function Attachments({
       );
     }
   });
-}
+};
+
+export default Attachments;