wip: Add helia and upload functionality

watcher/assets/styles/global.css

@@ -58,20 +58,6 @@ body {
   font-weight: bold;
 }
 
-.input-file {
-  position: relative;
-}
-
-.input-file input {
-  position: absolute;
-  top: 0;
-  left: 0;
-  width: 100%;
-  height: 100%;
-  opacity: 0;
-  cursor: pointer;
-}
-
 .input-file__text {
   display: flex;
   align-items: center;
@@ -108,6 +94,10 @@ body {
   font-size: 0.8rem;
 }
 
+.form .result.hidden {
+  display: none;
+}
+
 .form .result.error {
   background-color: #ef4444;
 }

watcher/helia.mjs

@@ -1,5 +1,15 @@
 import { createHelia } from 'helia'
+import { unixfs } from '@helia/unixfs'
+import { ipns } from '@helia/ipns'
 
-export async function createNode () {
+export async function createNode() {
   return await createHelia()
 }
+
+export async function createUnixFs(helia) {
+  return await unixfs(helia)
+}
+
+export async function createIPNS(helia) {
+  return await ipns(helia)
+}

watcher/main.js

@@ -1,16 +1,29 @@
 const path = require('path')
-const { app, BrowserWindow, shell } = require('electron')
+const fs = require('fs')
+const { app, BrowserWindow, shell, ipcMain, dialog } = require('electron')
 
-const gotTheLock = app.requestSingleInstanceLock()
 let mainWindow = null
+const gotTheLock = app.requestSingleInstanceLock()
+const data = {
+  node: null,
+  unixfs: null,
+  ipns: null
+}
+
+const uploadPlaylist = async (dir, filename) => {
+  const bytes = fs.readFileSync(path.join(dir, filename))
+  const cid = await data.unixfs.addBytes(bytes)
+  return cid
+}
 
 const createWindow = () => {
   mainWindow = new BrowserWindow({
     width: 800,
     height: 600,
     webPreferences: {
       preload: path.join(__dirname, './pages/preload.js'),
-      nodeIntegration: true
+      nodeIntegration: true,
+      enableRemoteModule: true
     }
   })
 
@@ -26,11 +39,167 @@ const createWindow = () => {
     return { action: 'deny' };
   });
 
-  if (process.env.NODE_ENV !== 'production') {
+  ipcMain.handle('dialog:openDirectory', async () => {
+    const { canceled, filePaths } = await dialog.showOpenDialog(mainWindow, {
+      properties: ['openDirectory']
+    })
+    if (canceled) {
+      return
+    } else {
+      return filePaths[0]
+    }
+  })
+
+  ipcMain.handle('upload:start', async (e, dir) => {
+    if (data.node === null) {
+      createHeliaNode()
+      return
+    }
+
+    try {
+      if (!dir) {
+        throw new Error()
+      }
+
+      const stat = await fs.promises.stat(dir)
+      if (!stat.isDirectory()) {
+        throw new Error()
+      }
+    } catch (err) {
+      return {
+        error: "Selected path is not a directory"
+      }
+    }
+
+    let playlistFilename = null
+    let playlistFile = null
+    const uploadedFiles = []
+    if (fs.readdirSync(dir).length !== 0) {
+      // sort files by created date
+      let fileNames = fs.readdirSync(dir)
+
+      playlistFilename = fileNames.find((filename) => {
+        return filename.endsWith('.m3u8')
+      })
+
+      if (playlistFilename) {
+        mainWindow.webContents.send("upload:message", { message: "found playlist file", playlistFilename })
+        fileNames = fileNames
+          .filter((filename) => filename !== playlistFilename)
+          .map((filename) => {
+            const stat = fs.statSync(path.join(dir, filename))
+            return {
+              filename,
+              stat
+            }
+          }).sort((a, b) => {
+            return a.stat.birthtimeMs - b.stat.birthtimeMs
+          })
+
+        mainWindow.webContents.send("upload:message", { message: "catching up with existing files" })
+        playlistFile = fs.readFileSync(path.join(dir, playlistFilename), 'utf8')
+        const files = fileNames.map(({ filename }) => {
+          return {
+            path: filename,
+            content: fs.readFileSync(path.join(dir, filename))
+          }
+        })
+
+        for await (const file of data.unixfs.addAll(files, { wrapWithDirectory: true })) {
+          mainWindow.webContents.send("upload:message", { message: "uploaded segment", segment: file.path, matchingCID: file.cid.toString()})
+          playlistFile = playlistFile.replace(file.path, file.cid.toString())
+          uploadedFiles.push(file.path)
+        }
+
+        fs.writeFileSync(path.join(dir, playlistFilename), playlistFile)
+        mainWindow.webContents.send("upload:message", { message: "updated playlist file", playlistFile })
+
+        const cid = await uploadPlaylist(dir, playlistFilename)
+        mainWindow.webContents.send("upload:message", { message: "uploaded playlist file", playlistFile: cid.toString() })
+        const tb = await import('it-to-buffer')
+        const newFilename = "./sample.m3u8"
+        const d = await tb.default(await data.unixfs.cat(cid.toString()))
+        fs.writeFileSync(newFilename, d)
+      }
+    }
+
+    mainWindow.webContents.send("upload:message", { message: "waiting for changes" })
+    fs.watch(dir, async (eventType, filename) => {
+      if (playlistFilename) {
+        if (eventType !== 'rename' || filename !== playlistFilename) {
+          return
+        }
+      } else {
+        if (filename.endsWith('.m3u8')) {
+          mainWindow.webContents.send("upload:message", { message: "playlist file is created", filename })
+          playlistFilename = filename
+          playlistFile = fs.readFileSync(path.join(dir, playlistFilename), 'utf8')
+          const files = fs.readdirSync(dir).map(({ filename }) => {
+            return {
+              path: filename,
+              content: fs.readFileSync(path.join(dir, filename))
+            }
+          })
+
+          mainWindow.webContents.send("upload:message", { message: "uploading files" })
+          for await (const file of data.unixfs.addAll(files, { wrapWithDirectory: true })) {
+            mainWindow.webContents.send("upload:message", { message: "uploaded segment", segment: file.path, matchingCID: file.cid.toString()})
+            playlistFile = playlistFile.replace(file.path, file.cid.toString())
+            uploadedFiles.push(file.path)
+          }
+
+          fs.writeFileSync(path.join(dir, playlistFilename), playlistFile)
+          mainWindow.webContents.send("upload:message", { message: "updated playlist file", playlistFile })
+
+          const cid = await uploadPlaylist(dir, playlistFilename)
+          mainWindow.webContents.send("upload:message", { message: "uploaded playlist file", playlistFile: cid.toString() })
+        }
+        return
+      }
+
+      playlistFile = fs.readFileSync(path.join(dir, playlistFilename), 'utf8')
+      const files = fs.readdirSync(dir)
+        .map(({ filename }) => {
+          return {
+            path: filename,
+            content: fs.readFileSync(path.join(dir, filename))
+          }
+        })
+
+      for await (const file of data.unixfs.addAll(files, { wrapWithDirectory: true })) {
+        mainWindow.webContents.send("upload:message", { message: "uploaded segment", segment: file.path, matchingCID: file.cid.toString()})
+        playlistFile = playlistFile.replace(file.path, file.cid.toString())
+        uploadedFiles.push(file.path)
+      }
+      fs.writeFileSync(path.join(dir, playlistFilename), playlistFile)
+      mainWindow.webContents.send("upload:message", { message: "updated playlist file", playlistFile })
+    })
+
+  })
+
+  if (process.env.NODE_ENV !== 'production' && process.env.DISABLE_DEVTOOLS.toLowerCase() !== 'true') {
     mainWindow.webContents.openDevTools()
   }
 }
 
+const createHeliaNode = async () => {
+  try {
+    // Helia is an ESM-only module but Electron currently only supports CJS
+    // at the top level, so we have to use dynamic imports to load it
+    const { createNode, createUnixFs, createIPNS } = await import('./helia.mjs')
+    data.node = await createNode()
+    const id = data.node.libp2p.peerId
+    data.unixfs = await createUnixFs(data.node)
+    data.ipns = await createIPNS(data.node)
+
+    if (process.env.NODE_ENV !== 'production') {
+      console.log(id)
+    }
+  } catch (err) {
+    console.error(err)
+  }
+}
+
 if (!gotTheLock) {
   app.quit()
 } else {
@@ -43,17 +212,7 @@ if (!gotTheLock) {
 
   app.whenReady().then(async () => {
     createWindow()
-
-    try {
-      // Helia is an ESM-only module but Electron currently only supports CJS
-      // at the top level, so we have to use dynamic imports to load it
-      const { createNode } = await import('./helia.mjs')
-      const node = await createNode()
-      const id = node.libp2p.peerId
-      console.log(id)
-    } catch (err) {
-      console.error(err)
-    }
+    createHeliaNode()
 
     app.on('activate', () => {
       if (BrowserWindow.getAllWindows().length === 0) createWindow()

watcher/package.json

@@ -8,6 +8,9 @@
     "start": "electron ./main.js"
   },
   "dependencies": {
-    "helia": "^1.3.12"
+    "@helia/ipns": "^1.1.3",
+    "@helia/unixfs": "^1.4.1",
+    "helia": "^1.3.12",
+    "it-to-buffer": "^4.0.2"
   }
 }

watcher/pages/index.html

@@ -15,12 +15,11 @@ <h1 class="title">Mjolnir Superhack</h1>
 
         <div class="form-container">
           <form class="form">
-            <p class="result error">There was an error</p>
+            <p class="result hidden"></p>
             <div class="input-group">
               <label for="file">Select OBS recording directory</label>
               <div class="input-file">
                 <div class="input-file__text">Select directory</div>
-                <input type="file" id="file" name="file" webkitdirectory directory multiple>
               </div>
             </div>
             <div class="actions">
@@ -51,6 +50,10 @@ <h1 class="title">Mjolnir Superhack</h1>
             </ol>
           </div>
         </div>
+
+        <div class="form-container">
+          <div class="log hidden"></div>
+        </div>
       </div>
     </div>
   </body>

watcher/pages/preload.js

@@ -0,0 +1,11 @@
+const { contextBridge, ipcRenderer } = require('electron')
+
+contextBridge.exposeInMainWorld('mapi', {
+  selectFolder: () => ipcRenderer.invoke('dialog:openDirectory'),
+  startUpload: (folder) => ipcRenderer.invoke('upload:start', folder),
+  onUploadMessage: (callback) => {
+    ipcRenderer.on('upload:message', function(evt, message) {
+      callback(message)
+    })
+  },
+})