Merge branch 'main' into fe-unit-test

.github/workflows/prevent-appconfig-push.yml

@@ -0,0 +1,18 @@
+name: Prevent AppConfig.plist Push
+
+on: [push, pull_request]
+
+jobs:
+  check-appconfig:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Check for AppConfig.plist
+        run: |
+          if git ls-files | grep -q 'AppConfig.plist'; then
+            echo "AppConfig.plist file detected. Failing the workflow."
+            exit 1
+          fi
+

.husky/pre-commit

@@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+sh .husky/pre-commit-script.sh

.husky/pre-commit-script.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# Check for AppConfig.plist
+if git diff --cached --name-only | grep --quiet "AppConfig.plist"; then
+  echo "Error: Commit contains AppConfig.plist, which is not allowed."
+  exit 1
+fi
+

package.json

@@ -0,0 +1,8 @@
+{
+  "name": "x",
+  "version": "1.0.0",
+  "private": true,
+  "devDependencies": {
+    "husky": "^8.0.3"
+  }
+}

progress/aaron-li.md

@@ -1,3 +1,7 @@
+2023-11-11 Sat (6h): Implement basic key protection in relay with AES encryption, multi-key rotation, device id and ip tracking, ban list, and key retrieval API with rate limit; Implement client side decryption and initialization; Client side bug fixes, end-to-end testing; Deployment on GCP and systemd service with instance metadata as parameter; Setup and build guide; Allow local key override
+
+2023-11-10 Fri (1.5h): Review #141, #157, #159, #160; Research on GPT chat session, context compression and effects; Interview preparation and discussion on technical problems 
+
 2023-11-09 Thu (5h): Review #144, #145, #146, #148, #149, #150, #151, #152; Implement QPM limit on streaming services, and rate-limit error handling; Test end-to-end; Research and analysis on iOS market shares by versions, and minimally required versions for key features; Code refactor and applying universal formatting; Implement GPT model switch based on GPT usage time; Implement random alert to share and review based on app usage time; Test end-to-end and fix bugs
 
 2023-11-08 Wed (4h): Review #138, #139; Testing for bugs in complex audio conditions; Update production checklist; Review language requirement, support channel integration, built-in analytics; Review and discussion on checklist suggestions; Technical interview preparation; Extend production checklist with audio bugs, UI bugs, and customer support features; Cleanup and backup Notion data; Fix issues on mail aliases services, implemented pagination; Update checklist on version control issues; Review deep links into settings;
@@ -6,8 +10,6 @@
 
 2023-11-06 Mon (2h): Review #114, #115, #117 update, #118, #119, #120, #121, #123, #125; Review OpenAI new offerings and ways of integrations.
 
----
-
 2023-11-03 Fri (0.5h): Review #117; New domain and DNS configuration; Review Linux ML development portability to iOS.
 
 2023-11-02 Thu (1h): DNS configurations for lend and Cloudflare issue diagnosis; Lend legal term review; Discussions with security vendors on manual review of swap; Review on-device models and compatibility with iOS devices.

progress/amanda-bancroft.md

@@ -0,0 +1,30 @@
+2023-11-13 Mon: 
+
+---
+
+2023-11-10 Fri: 
+I've noticed that there are instances where I need to repeat my question to prompt a response from the app. In a particular interaction, I asked the app to list the top five restaurants in an area. At the end of the list, when I inquired about the fourth restaurant, I had to ask multiple times before the app eventually referred to the correct entry. In the same session, after discussing various topics and referencing that part of the conversation, the app did not repeat the correct information that I requested. In another instance, when I asked about the pricing at a botanical garden, the app provided the information but mentioned it might not be up to date. When I inquired about the latest year of data, the app avoided answering the question. Additionally, when the app referenced a restaurant's website during our conversation, and I asked for the website, it simply advised me to perform a Google search with the restaurant's name and 'official website'.
+
+2023-11-09 Thurs:
+The app provided a long detailed response to one of my questions, prompting me to request a quicker listing of names. Although it complied promptly, I had to ask the app to slow down two separate times and spell out the names twice for clarity. I find response time and rate of speech to be fairly inconsistent. 
+
+2023-11-08 Wed:
+I experienced delays in response, and the voice doesn't quite meet my preferences. Consider categorizing fun facts based on user interests for a more personalized experience, using data from the current conversation. Additionally, it would enhance user convenience if the user guide were presented as a pop-up within the app rather than linking to an external webpage.
+
+I came across a bug where I had to hold the 'Surprise ME!' button. Additionally, there's a suggestion to consider changing all buttons to lowercase to align with current trends. When I inquired about Los Angeles, the assistant responded appropriately, but when I asked for more details about places to visit, I received the exact same lengthy response. On a positive note, the 'New Session' button works seamlessly.
+
+2023-11-07 Tues:
+While using the app, I often find myself wishing for the ability to engage with it in a hands-free manner. The allure of a speech-driven application lies in its effortless operation, allowing for a more natural conversation and the ability to use the app while engaged in other activities like washing dishes or driving. It would be convenient not to require pressing and holding a button to speak.
+
+2023-11-06 Mon:
+I envision using this app primarily for planning and brainstorming, be it for meal preparation, travel itineraries, or daily routines. Nevertheless, a significant drawback lies in its inability to deliver real-time data like traffic updates, weather forecasts, current time, and the most precise and up-to-date information on restaurants, stores, and other establishments.
+
+---
+
+2023-11-03 Fri:
+The app has an elegant and uncomplicated design. While the 'random fact' feature may not need to take center stage, incorporating a small text box for easy copying and pasting of relevant information would be a valuable addition. The app excels in its user-friendly interface. Furthermore, introducing diverse voice options, similar to those found in map applications, would be a noteworthy improvement.
+
+2023-11-02 Thurs:
+I used the app to explore various dinner options. I appreciated the convenience of engaging in a conversation rather than sifting through online recipes. The app offered simple yet tailored meal suggestions based on my dietary preferences and additional queries. I also used the app to plan out a week's worth of meals. I can see myself making regular use of this app for brainstorming and meals. One minor issue I encountered was occasional delays in response when asking a direct question, requiring me to re-prompt the app.
+
+It would enhance convenience if there were an option for a text pop-up when specific questions are asked, like 'create a grocery list.' This way, users wouldn't have to manually transcribe the request, making it as simple as copy and paste. I found the app's initial use to be straightforward, though its emphasis on mega-minimalism might be initially surprising for some, but likely easy to adapt to. Consider enlarging the main functions, such as 'speak,' 'pause,' and 'repeat,' for better accessibility, while keeping the less frequently used functions smaller.

progress/julia-nai.md

@@ -1,3 +1,5 @@
+2023-11-10 Fri: Engaged in product testing, investigating the handling of overlap with the tap/hold buttons. When both are activated, the other occasionally does not turn off. This seems to be related to the way stop transcribing is handled. [Assisted](https://github.com/harmony-one/x/commit/73204359eae8079c968d531fd0333591db373642) in reverting code that was causing some compile issues. Triaging issues related to .gitignore. [Merged](https://github.com/harmony-one/x/pull/163) and reviewed #163 Add updates to unit tests.
+
 2023-11-09 Thu: [Created](https://docs.google.com/spreadsheets/d/1IlE0EpLUsmmDPwdx2b6adzw1UGeY3f-o-dO6qbWN0b8/edit?usp=sharing) dynamic pricing calculator spreadsheet for GPT 4 with adjustable assumptions. 
 [Reverted](https://github.com/harmony-one/x/commit/0659298ca8caca81ec4c34f26ce3e2fd05a4c897) custom instructions.
 Worked on investigating the tap to speak button slowness. I have a local build which dumps extra debugging information in an overly verbose way, but I have not been able to identify the source of the issue yet. 

progress/rikako-hatoya.md

@@ -1,6 +1,10 @@
+2023-11-11 Sat: Further implemented a unit test for TextToSpeechConverterTests to handle convertTextToSpeech() when provided device language is not available in the AVSpeechSynthesisVoice framework. Submitted PR: [https://github.com/harmony-one/x/pull/164]. Worked on unit test when voice is available for the language.
+
+2023-11-10 Fri: Investigated debugging "IsFormatSampleRateAndChannelCountValid(format)". Implemented unit tests for TextToSpeechConverter, made a mock class for AVSpeechSynthesizer, updated TextToSpeechConverter by adding a protocol, and added code to make test debugging easier at breakpoints. Raised the overall test coverage for textToSpeech to a 93%, submitted PR: [https://github.com/harmony-one/x/pull/163].
+
 2023-11-09 Thu: Debugging for bugs caused by unit tests requesting microphone permission (issue due to "IsFormatSampleRateAndChannelCountValid(format)" causes entire test to fail due to thread kill). Currently still working on it but so far: 1. Improved error handling for audioSession.recordPermission(). 2. Added NS microphone permissions requestg in Info.plist 3. Set up AVAudioSession properly by setting setCategory() and setActive() for AVAudioSession() 
 
-2023-11-08 Wed: Resolved haptics issue which was earlier colliding with recording functions. Implemented language support so that app defaults to device's preferred first language instead of English. Debugging on compiling issues for running unit tests.
+2023-11-08 Wed: Resolved haptics issue which was earlier colliding with recording functions (PR: [https://github.com/harmony-one/x/pull/141]). Implemented language support so that app defaults to device's preferred first language instead of English (PR: [https://github.com/harmony-one/x/pull/143]). Debugging on compiling issues for running unit tests.
 
 2023-11-07 Tue: Investigated a method to mitigate the haptics issue using UIImpactFeedbackGenerator and modifying SpeechRecognition.swift (Haptics feedback using AudioServicesPlayAlertSound() had bug: conflict with AVAudio when functions .playAndRecord() or .record() are in action). Investigated unit test failures due to new functions being implemented, which were leading to threading errors. Assisted Theo with preparation for the TGI event.
 

progress/sun-hyuk-ahn.md

@@ -1,10 +1,16 @@
-2023-11-10 Thu: Implementing long press actions for multiple buttons. Fixing various bugs for the launch day. Debugging Aaron's rate limit merge (initially defaulted to 3.5turbo due to date calculation bug).
+2023-11-12 Sun: Reviewed Aaron's PR on relayer service, as well as the configuration in the cloud service.
 
-2023-11-09 Wed: Implemented "Press to Speak & Press to Send" button. Researched into different tools we can use to develop the "share" feature. [Branch](https://www.branch.io/) seems to be the most useful SDK. Began looking into the SDK to start implementation.
+2023-11-11 Sat: Configured [Github action and Husky pre-commit hook](https://github.com/harmony-one/x/pull/165) preventing AppConfig.plist from being committed.
 
-2023-11-08 Tue: Update streaming to initially flush 5 words then by delimiting punctuations, upperbounded by 20 words. Finished setting up Sentry with Yuriy. Exponential backoff changes to 2, 4, 8 seconds.
+2023-11-10 Fri: Debugged time calculation logic that triggers gpt-3.5.
 
-2023-11-07 Mon: Fixed a bug which ensures streams that are part of the previous word is appended correctly. Upperbounded buffer to contain 10 words at max so that synthesis occurs at delimiter or 10 words. Implemented retryable methods with exponential backoff.
+2023-11-09 Thu: Implementing long press actions for multiple buttons. Fixing various bugs for the launch day. Debugging Aaron's rate limit merge (initially defaulted to 3.5turbo due to date calculation bug).
+
+2023-11-08 Wed: Implemented "Press to Speak & Press to Send" button. Researched into different tools we can use to develop the "share" feature. [Branch](https://www.branch.io/) seems to be the most useful SDK. Began looking into the SDK to start implementation.
+
+2023-11-07 Tue: Update streaming to initially flush 5 words then by delimiting punctuations, upperbounded by 20 words. Finished setting up Sentry with Yuriy. Exponential backoff changes to 2, 4, 8 seconds.
+
+2023-11-06 Mon: Fixed a bug which ensures streams that are part of the previous word is appended correctly. Upperbounded buffer to contain 10 words at max so that synthesis occurs at delimiter or 10 words. Implemented retryable methods with exponential backoff.
 
 ---
 

progress/theo-perisic.md

@@ -1,3 +1,9 @@
+2023-11-12 Sun: Reviewed Fenwick Terms of Service, sent comments back for changes.
+
+2023-11-11 Sat: Continued testing Voice AI, currently heavy burden on user to get value out of usage.
+
+2023-11-10 Fri: Completed an initial release usage and [feedback table](https://harmonyone.notion.site/Initial-Release-Usage-and-Feedback-661d427191534b6f9a58e950685e1242?pvs=4) to focus on main blockers to “30 people 30 minutes” goal.
+
 2023-11-09 Thu: Following App Store launch, completed [in-app growth strategies](https://harmonyone.notion.site/App-Growth-Brainstorm-5a776417f42d4c38923bf541a79df413?pvs=4). Continued [app testing](https://harmonyone.notion.site/App-Store-Release-Testing-f4c1cc052fc84d999a6ddd8e218fb27a?pvs=4). 
 
 2023-11-08 Wed: Completed [Product Marketing Strategy](https://harmonyone.notion.site/Voice-AI-Marketing-Product-Strategy-7034a6c564124d23be4f061def94eaff?pvs=4) draft and added [custom instructions](https://harmonyone.notion.site/Custom-Instructions-2-Voice-AI-a14b176b252d4c539f7635c8777c5ece?pvs=4) optimized for user experience. Started competitor analysis page using MobileAction to track App Store Optimization.

voice/relay/deploy/enable.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+sudo setcap CAP_NET_BIND_SERVICE=+eip /usr/bin/node
+sudo cp voice-ai-relay.service /etc/systemd/system/voice-ai-relay.service
+sudo systemctl start voice-ai-relay
+sudo systemctl enable voice-ai-relay
+systemctl status voice-ai-relay

voice/relay/deploy/log.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+journalctl -u voice-ai-relay -n 1000 -f

voice/relay/deploy/port.sh

@@ -0,0 +1,8 @@
+!#/bin/sh
+sudo iptables -A INPUT -i eth0 -p tcp --dport 80 -j ACCEPT
+sudo iptables -A INPUT -i eth0 -p tcp --dport 3000 -j ACCEPT
+sudo iptables -A INPUT -i eth0 -p tcp --dport 443 -j ACCEPT
+sudo iptables -A INPUT -i eth0 -p tcp --dport 8443 -j ACCEPT
+sudo iptables -A PREROUTING -t nat -i eth0 -p tcp --dport 80 -j REDIRECT --to-port 3000
+sudo iptables -A PREROUTING -t nat -i eth0 -p tcp --dport 443 -j REDIRECT --to-port 8443
+sudo iptables --flush

voice/relay/deploy/voice-ai-relay.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=Voice AI Relay Server
+Documentation=https://github.com/harmony-one/x/
+After=network.target
+
+[Service]
+Environment=PORT=80 HTTPS_PORT=443
+Type=simple
+User=worker
+WorkingDirectory=/opt/git/x/voice/relay
+ExecStart=/bin/bash -c "source ~/.profile;/usr/bin/node --loader ts-node/esm ./bin/run.ts"
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target

voice/relay/package.json

@@ -12,20 +12,26 @@
   "dependencies": {
     "@simplewebauthn/server": "^8.3.2",
     "@types/cookie-parser": "^1.4.4",
+    "@types/request-ip": "^0.0.41",
     "axios": "^1.5.1",
     "cbor": "^9.0.1",
+    "compression": "^1.7.4",
     "cookie-parser": "^1.4.6",
     "cookie-session": "^2.0.0",
     "cors": "^2.8.5",
     "dotenv": "^16.3.1",
     "express": "^4.18.2",
+    "express-rate-limit": "^7.1.4",
     "fast-sha256": "^1.3.0",
+    "http-errors": "^2.0.0",
     "jsrsasign": "^10.8.6",
     "morgan": "^1.10.0",
     "node-cache": "^5.1.2",
-    "openai": "^4.14.0"
+    "openai": "^4.14.0",
+    "request-ip": "^3.3.0"
   },
   "devDependencies": {
+    "@types/compression": "^1.7.5",
     "@types/cookie": "^0.5.2",
     "@types/cookie-session": "^2.0.45",
     "@types/cors": "^2.8.14",

voice/relay/src/app.ts

@@ -1,12 +1,16 @@
-import express from 'express'
+import express, { type Response, type Request, type NextFunction } from 'express'
 import cookieParser from 'cookie-parser'
 import morgan from 'morgan'
-import apiRouter from './routes/index.js'
+import createError from 'http-errors'
+// import apiRouter from './routes/hard.js'
+import softRateLimitedApiRouter from './routes/soft.js'
 import fs from 'fs'
 import http, { type Server as HttpServer } from 'http'
 import https from 'https'
 import config from './config/index.js'
 import cors from 'cors'
+import compression from 'compression'
+import requestIp from 'request-ip'
 
 const app = express()
 let httpServer: HttpServer
@@ -30,6 +34,8 @@ if (config.https.only) {
   httpServer = http.createServer(app)
 }
 const httpsServer = https.createServer(httpsOptions, app)
+app.use(compression())
+app.use(requestIp.mw())
 
 app.use(morgan('common'))
 app.use(cookieParser())
@@ -58,7 +64,23 @@ app.options('*', async (_req, res) => {
   res.end()
 })
 
-app.use('/', apiRouter)
+app.use('/soft', softRateLimitedApiRouter)
+
+// catch 404 and forward to error handler
+app.use(function (req, res, next) {
+  next(createError(404))
+})
+
+// error handler
+app.use((err: Error, req: Request, res: Response, next: NextFunction) => {
+  // set locals, only providing error in development
+  res.locals.message = err.message
+  res.locals.error = config.debug ? err : ''
+
+  // render the error page
+  res.status(500)
+  res.json({ error: res.locals.error, message: err.message, stack: config.debug ? err.stack : undefined })
+})
 
 export {
   httpServer,

voice/relay/src/config/index.ts

@@ -15,7 +15,14 @@ const config = {
   packageName: process.env.PACKAGE_NAME ?? '',
   openai: { key: process.env.OPENAI_KEY ?? '' },
   deepgram: { key: process.env.DEEPGRAM_KEY ?? '' },
-  playht: { key: process.env.PLAYHT_KEY ?? '' },
+  playht: { key: process.env.PLAYHT_KEY ?? '' }
 }
 
+export const OpenAIDistributedKeys: string[] = JSON.parse(process.env.OPENAI_DISTRIBUTED_KEYS ?? '[]')
+export const BlockedDeviceIds: string[] = JSON.parse(process.env.BLOCKED_DEVICE_IDS ?? '[]')
+export const BlockedIps: string[] = JSON.parse(process.env.BLOCKED_IPS ?? '[]')
+
+export const SharedEncryptionSecret: string = process.env.SHARED_ENCRYPTION_SECRET ?? ''
+export const SharedEncryptionIV: string = process.env.SHARED_ENCRYPTION_IV ?? ''
+
 export default config