feat: AI text to speech for proposal body (#244)

* feat: AI text to speech for proposal body * fix: add min proposal body length constraint * fix: return meaningful error message when proposal is not supported * fix: strip markdown before sending to speech AI * fix: fix error thrown when instantiating openAI with empty API KEY * fix: return a more precise error code * fix: fix invalid env var name --------- Co-authored-by: ChaituVR <[email protected]>
snapshot-labs · Mar 29, 2024 · 5511e22 · 5511e22
1 parent a504350
commit 5511e22
Show file tree

Hide file tree

Showing 6 changed files with 109 additions and 16 deletions.
diff --git a/.env.example b/.env.example
@@ -9,6 +9,9 @@ AWS_BUCKET_NAME=
 WEBHOOK_AUTH_TOKEN=
 STORAGE_ENGINE=file
 VOTE_REPORT_SUBDIR=votes
+AI_SUMMARY_SUBDIR=ai-summary
+AI_TTS_SUBDIR=ai-tts
+OPENAI_API_KEY=
 NFT_CLAIMER_PRIVATE_KEY=
 NFT_CLAIMER_NETWORK=
 NFT_CLAIMER_DEPLOY_VERIFYING_CONTRACT=

diff --git a/package.json b/package.json
@@ -48,7 +48,8 @@
     "multiformats": "^9",
     "mysql": "^2.18.1",
     "node-fetch": "^2.7.0",
-    "openai": "^4.29.2"
+    "openai": "^4.29.2",
+    "remove-markdown": "^0.5.0"
   },
   "devDependencies": {
     "@snapshot-labs/eslint-config": "^0.1.0-beta.15",
@@ -61,6 +62,7 @@
     "@types/mysql": "^2.15.21",
     "@types/node": "^20.4.8",
     "@types/node-fetch": "^2.6.4",
+    "@types/remove-markdown": "^0.3.4",
     "@types/supertest": "^2.0.12",
     "@typescript-eslint/eslint-plugin": "^6.7.2",
     "copyfiles": "^2.4.1",

diff --git a/src/api.ts b/src/api.ts
@@ -7,7 +7,8 @@ import mintPayload from './lib/nftClaimer/mint';
 import deployPayload from './lib/nftClaimer/deploy';
 import { queue, getProgress } from './lib/queue';
 import { snapshotFee } from './lib/nftClaimer/utils';
-import AISummary from './lib/ai/summary';
+import AiSummary from './lib/ai/summary';
+import AiTextToSpeech from './lib/ai/textToSpeech';
 
 const router = express.Router();
 
@@ -40,7 +41,7 @@ router.all('/votes/:id', async (req, res) => {
 
 router.post('/ai/summary/:id', async (req, res) => {
   const { id } = req.params;
-  const aiSummary = new AISummary(id, storageEngine(process.env.AI_SUMMARY_SUBDIR));
+  const aiSummary = new AiSummary(id, storageEngine(process.env.AI_SUMMARY_SUBDIR));
 
   try {
     const cachedSummary = await aiSummary.getCache();
@@ -54,9 +55,38 @@ router.post('/ai/summary/:id', async (req, res) => {
     }
 
     return rpcSuccess(res.status(200), summary, id);
-  } catch (e) {
+  } catch (e: any) {
     capture(e);
-    return rpcError(res, 'INTERNAL_ERROR', id);
+    return rpcError(res, e.message || 'INTERNAL_ERROR', id);
+  }
+});
+
+router.post('/ai/tts/:id', async (req, res) => {
+  const { id } = req.params;
+  const aiTextTpSpeech = new AiTextToSpeech(id, storageEngine(process.env.AI_TTS_SUBDIR));
+
+  try {
+    const cachedAudio = await aiTextTpSpeech.getCache();
+
+    let audio: Buffer;
+
+    if (!cachedAudio) {
+      try {
+        audio = (await aiTextTpSpeech.createCache()) as Buffer;
+      } catch (e: any) {
+        capture(e);
+        return rpcError(res, e, id);
+      }
+    } else {
+      audio = cachedAudio as Buffer;
+    }
+
+    res.header('Content-Type', 'audio/mpeg');
+    res.attachment(aiTextTpSpeech.filename);
+    return res.end(audio);
+  } catch (e: any) {
+    capture(e);
+    return rpcError(res, e.message || 'INTERNAL_ERROR', id);
   }
 });
 

diff --git a/src/lib/ai/summary.ts b/src/lib/ai/summary.ts
@@ -1,24 +1,23 @@
 import OpenAI from 'openai';
-import { capture } from '@snapshot-labs/snapshot-sentry';
 import { fetchProposal, Proposal } from '../../helpers/snapshot';
 import { IStorage } from '../storage/types';
 import Cache from '../cache';
 
-const openai = new OpenAI({ apiKey: process.env.apiKey });
-
-class AISummary extends Cache {
+class Summary extends Cache {
   proposal?: Proposal | null;
+  openAi: OpenAI;
 
   constructor(id: string, storage: IStorage) {
     super(id, storage);
     this.filename = `snapshot-proposal-ai-summary-${this.id}.txt`;
+    this.openAi = new OpenAI({ apiKey: process.env.OPENAI_API_KEY || 'Missing key' });
   }
 
   async isCacheable() {
     this.proposal = await fetchProposal(this.id);
 
     if (!this.proposal) {
-      return Promise.reject('RECORD_NOT_FOUND');
+      throw new Error('RECORD_NOT_FOUND');
     }
 
     return true;
@@ -30,7 +29,7 @@ class AISummary extends Cache {
     try {
       const { body, title, space } = this.proposal!;
 
-      const completion = await openai.chat.completions.create({
+      const completion = await this.openAi.chat.completions.create({
         messages: [
           {
             role: 'system',
@@ -41,20 +40,19 @@ class AISummary extends Cache {
       });
 
       if (completion.choices.length === 0) {
-        throw new Error('No completion in response');
+        throw new Error('EMPTY_OPENAI_CHOICES');
       }
       const content = completion.choices[0].message.content;
 
       if (!content) {
-        throw new Error('No content in response');
+        throw new Error('EMPTY_OPENAI_RESPONSE');
       }
 
       return content;
     } catch (e: any) {
-      capture(e);
-      throw e;
+      throw e.error?.code ? new Error(e.error?.code.toUpperCase()) : e;
     }
   };
 }
 
-export default AISummary;
+export default Summary;
diff --git a/src/lib/ai/textToSpeech.ts b/src/lib/ai/textToSpeech.ts
@@ -0,0 +1,50 @@
+import OpenAI from 'openai';
+import removeMd from 'remove-markdown';
+import Cache from '../cache';
+import { fetchProposal, Proposal } from '../../helpers/snapshot';
+import { IStorage } from '../storage/types';
+
+const MIN_BODY_LENGTH = 500;
+const MAX_BODY_LENGTH = 4096;
+
+export default class TextToSpeech extends Cache {
+  proposal?: Proposal | null;
+  openAi: OpenAI;
+
+  constructor(id: string, storage: IStorage) {
+    super(id, storage);
+    this.filename = `snapshot-proposal-ai-tts-${this.id}.mp3`;
+    this.openAi = new OpenAI({ apiKey: process.env.OPENAI_API_KEY || 'Missing key' });
+  }
+
+  async isCacheable() {
+    this.proposal = await fetchProposal(this.id);
+
+    if (!this.proposal) {
+      throw new Error('RECORD_NOT_FOUND');
+    }
+
+    return true;
+  }
+
+  getContent = async () => {
+    this.isCacheable();
+    const body = removeMd(this.proposal!.body);
+
+    if (body.length < MIN_BODY_LENGTH || body.length > MAX_BODY_LENGTH) {
+      throw new Error('UNSUPPORTED_PROPOSAL');
+    }
+
+    try {
+      const mp3 = await this.openAi.audio.speech.create({
+        model: 'tts-1',
+        voice: 'alloy',
+        input: body
+      });
+
+      return Buffer.from(await mp3.arrayBuffer());
+    } catch (e: any) {
+      throw e.error?.code ? new Error(e.error?.code.toUpperCase()) : e;
+    }
+  };
+}
diff --git a/yarn.lock b/yarn.lock
@@ -2352,6 +2352,11 @@
   resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc"
   integrity sha512-EEhsLsD6UsDM1yFhAvy0Cjr6VwmpMWqFBCb9w07wVugF7w9nfajxLuVmngTIpgS6svCnm6Vaw+MZhoDCKnOfsw==
 
+"@types/remove-markdown@^0.3.4":
+  version "0.3.4"
+  resolved "https://registry.yarnpkg.com/@types/remove-markdown/-/remove-markdown-0.3.4.tgz#b90c2e7117fbb30e51d5d849afac744715db4f91"
+  integrity sha512-i753EH/p02bw7bLlpfS/4CV1rdikbGiLabWyVsAvsFid3cA5RNU1frG7JycgY+NSnFwtoGlElvZVceCytecTDA==
+
 "@types/semver@^7.5.0":
   version "7.5.3"
   resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.5.3.tgz#9a726e116beb26c24f1ccd6850201e1246122e04"
@@ -5973,6 +5978,11 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/rehackt/-/rehackt-0.0.3.tgz#1ea454620d4641db8342e2db44595cf0e7ac6aa0"
   integrity sha512-aBRHudKhOWwsTvCbSoinzq+Lej/7R8e8UoPvLZo5HirZIIBLGAgdG7SL9QpdcBoQ7+3QYPi3lRLknAzXBlhZ7g==
 
+remove-markdown@^0.5.0:
+  version "0.5.0"
+  resolved "https://registry.yarnpkg.com/remove-markdown/-/remove-markdown-0.5.0.tgz#a596264bbd60b9ceab2e2ae86e5789eee91aee32"
+  integrity sha512-x917M80K97K5IN1L8lUvFehsfhR8cYjGQ/yAMRI9E7JIKivtl5Emo5iD13DhMr+VojzMCiYk8V2byNPwT/oapg==
+
 require-directory@^2.1.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42"