Merge pull request #686 from sparrowapp-dev/development

merge to release v2

Author: LordNayan

.env.example

@@ -89,6 +89,7 @@ AZURE_OPENAI_API_VERSION=         # API version (e.g., "2023-05-15")
 AZURE_OPENAI_MAX_TOKENS=          # Maximum tokens per AI response
 AZURE_OPENAI_MONTHLY_TOKEN_LIMIT= # Monthly token quota for AI services
 AZURE_OPENAI_ASSISTANT_ID=        # OpenAI assistant ID
+ENCRYPTION_SECRET=                # Encryption key for LLM key's Encryption
 
 # Deepseek AI Configuration
 DEEPSEEK_ENDPOINT=                # Deepseek API endpoint

migrations/auth-to-array.migration.ts

@@ -0,0 +1,123 @@
+import { Inject, Injectable, OnModuleInit } from "@nestjs/common";
+import { Db, ObjectId } from "mongodb";
+import { v4 as uuidv4 } from "uuid";
+import { Collections } from "@src/modules/common/enum/database.collection.enum";
+
+@Injectable()
+export class AuthToAuthProfilesMigration implements OnModuleInit {
+  private hasRun = false;
+
+  constructor(@Inject("DATABASE_CONNECTION") private db: Db) {}
+
+  async onModuleInit(): Promise<void> {
+    if (this.hasRun) return;
+
+    try {
+      console.log(`\n\x1b[32m[Nest]\x1b[0m \x1b[32mRunning AuthToAuthProfilesMigration...`);
+
+      const collection = this.db.collection(Collections.COLLECTION);
+
+      const documents = await collection
+        .find({
+          $or: [
+            { auth: { $type: "object" }, "auth.0": { $exists: false } }, // Valid auth object (not array)
+            { auth: { $exists: false } }, // No auth field
+            { auth: null }, // Null auth
+          ],
+        })
+        .toArray();
+
+      if (documents.length === 0) {
+        console.log("No documents needing authProfiles migration.");
+        this.hasRun = true;
+        return;
+      }
+
+      const now = new Date();
+
+      for (const doc of documents) {
+        const authObject = doc.auth;
+
+        // Case 1: auth is missing or invalid
+        if (!authObject || typeof authObject !== "object" || Array.isArray(authObject)) {
+          const update: any = {};
+
+          if (!Array.isArray(doc.authProfiles)) {
+            update.$set = { authProfiles: [] };
+
+            await collection.updateOne(
+              { _id: new ObjectId(doc._id) },
+              update
+            );
+
+            console.log(`Set empty authProfiles for document: ${doc._id}`);
+          } else {
+            console.log(`authProfiles already exists for document: ${doc._id}`);
+          }
+
+          continue;
+        }
+
+        // Case 2: auth is a valid object, migrate to authProfiles
+        let authType = "";
+        const { bearerToken, basicAuth = {}, apiKey = {} } = authObject;
+
+        const hasBearer = bearerToken && bearerToken.trim() !== "";
+        const hasBasic = basicAuth.username?.trim() || basicAuth.password?.trim();
+        const hasApiKey = apiKey.authKey?.trim() || apiKey.authValue?.trim();
+
+        if (hasBearer) {
+          authType = "Bearer Token";
+        } else if (hasBasic) {
+          authType = "Basic Auth";
+        } else if (hasApiKey) {
+          authType = "API Key";
+        } else {
+          console.warn(`Skipping ${doc._id}: no valid auth data.`);
+          continue;
+        }
+
+        const newAuthProfile = {
+          name: "New-Auth-Profile",
+          description: "",
+          authType,
+          auth: {
+            bearerToken: bearerToken || "",
+            basicAuth: {
+              username: basicAuth.username || "",
+              password: basicAuth.password || "",
+            },
+            apiKey: {
+              authKey: apiKey.authKey || "",
+              authValue: apiKey.authValue || "",
+              addTo: apiKey.addTo || "Header",
+            },
+          },
+          defaultKey: false,
+          createdAt: now,
+          authId: uuidv4(),
+        };
+
+        const update: any = {};
+
+        if (Array.isArray(doc.authProfiles)) {
+          update.$push = { authProfiles: newAuthProfile };
+        } else {
+          update.$set = { authProfiles: [newAuthProfile] };
+        }
+
+        await collection.updateOne(
+          { _id: new ObjectId(doc._id) },
+          update
+        );
+
+        console.log(`Updated authProfiles for document: ${doc._id}`);
+      }
+
+      console.log(`Migration completed. Total processed: ${documents.length}`);
+      this.hasRun = true;
+    } catch (error) {
+      console.error("Error during AuthToAuthProfilesMigration:", error);
+    }
+  }
+}

migrations/empty-array.migration.ts

@@ -0,0 +1,48 @@
+import { Inject, Injectable, OnModuleInit } from "@nestjs/common";
+import { Db, ObjectId } from "mongodb";
+import { Collections } from "@src/modules/common/enum/database.collection.enum";
+
+@Injectable()
+export class AuthToAuthProfilesMigration implements OnModuleInit {
+  private hasRun = false;
+
+  constructor(@Inject("DATABASE_CONNECTION") private db: Db) {}
+
+  async onModuleInit(): Promise<void> {
+    if (this.hasRun) return;
+
+    try {
+      console.log(`\n\x1b[32m[Nest]\x1b[0m \x1b[32mRunning AddEmptyAuthProfilesMigration...`);
+
+      const collection = this.db.collection(Collections.COLLECTION);
+
+      const documents = await collection
+        .find({
+          $or: [
+            { authProfiles: { $exists: false } },
+            { authProfiles: { $not: { $type: "array" } } },
+          ],
+        })
+        .toArray();
+
+      if (documents.length === 0) {
+        console.log("No documents needing authProfiles initialization.");
+        this.hasRun = true;
+        return;
+      }
+
+      for (const doc of documents) {
+        await collection.updateOne(
+          { _id: new ObjectId(doc._id) },
+          { $set: { authProfiles: [] } }
+        );
+        console.log(`Set empty authProfiles for document: ${doc._id}`);
+      }
+
+      console.log(`Migration completed. Total updated: ${documents.length}`);
+      this.hasRun = true;
+    } catch (error) {
+      console.error("Error during AddEmptyAuthProfilesMigration:", error);
+    }
+  }
+}

migrations/encrypt-llm.migration.ts

@@ -0,0 +1,79 @@
+import { Inject, Injectable, OnModuleInit } from '@nestjs/common';
+import { Db, ObjectId } from 'mongodb';
+import { ConfigService } from '@nestjs/config';
+import * as CryptoJS from 'crypto-js';
+
+@Injectable()
+export class EncryptLlmApiKeysMigration  implements OnModuleInit {
+  private hasRun = false;
+  private readonly secret: string;
+  private readonly iv = CryptoJS.enc.Hex.parse('00000000000000000000000000000000');
+
+  constructor(
+    @Inject('DATABASE_CONNECTION') private readonly db: Db,
+    private readonly configService: ConfigService,
+  ) {
+    this.secret = this.configService.get('ai.encryptionSecret');
+  }
+
+  async onModuleInit(): Promise<void> {
+    if (this.hasRun) return;
+
+    try {
+      console.log('\x1b[34m[Migration]\x1b[0m Encrypting API keys in llmconversation...');
+
+      const llmConversationCollection = this.db.collection('llmconversation');
+      const fields = ['openai', 'anthropic', 'deepseek', 'google'];
+
+      const documents = await llmConversationCollection.find({}).toArray();
+
+      for (const doc of documents) {
+        const updates: Record<string, any> = {};
+        let needsUpdate = false;
+
+        for (const field of fields) {
+          if (Array.isArray(doc[field])) {
+            const updatedArray = doc[field].map((entry: any) => {
+              if (entry?.value && !this.isEncrypted(entry.value)) {
+                const encrypted = this.encrypt(entry.value);
+                needsUpdate = true;
+                return { ...entry, value: encrypted };
+              }
+              return entry;
+            });
+            updates[field] = updatedArray;
+          }
+        }
+
+        if (needsUpdate) {
+          await llmConversationCollection.updateOne(
+            { _id: new ObjectId(doc._id) },
+            { $set: updates },
+          );
+          console.log(`Updated document _id: ${doc._id}`);
+        }
+      }
+
+      this.hasRun = true;
+      console.log('\x1b[32m[Migration]\x1b[0m Encryption completed successfully.');
+    } catch (err) {
+      console.error('[Migration Error]', err);
+    }
+  }
+
+  private encrypt(text: string): string {
+    const key = CryptoJS.enc.Utf8.parse(this.secret.padEnd(32, ' '));
+    const encrypted = CryptoJS.AES.encrypt(text, key, {
+      iv: this.iv,
+      mode: CryptoJS.mode.CBC,
+      padding: CryptoJS.pad.Pkcs7,
+    });
+
+    return encrypted.ciphertext.toString(CryptoJS.enc.Base64);
+  }
+
+  private isEncrypted(value: string): boolean {
+    // Basic check: OpenAI key pattern usually starts with 'sk-'
+    return /^[A-Za-z0-9+/=]{32,}$/.test(value) && !value.startsWith('sk-');
+  }
+}

migrations/update-hub-billing.migration.ts

@@ -73,8 +73,6 @@ export class UpdateHubBillingMigration implements OnModuleInit {
         amount_billed: 119.88,
         currency: "usd",
         status: "active",
-        collection_method: "charge_manually",
-        paid_at: new Date("2025-07-08T13:04:28.000Z"),
         billingType: "paid",
         updatedBy: "system-admin",
         in_trial: false,

package.json

@@ -43,6 +43,7 @@
     "@azure/service-bus": "^7.9.1",
     "@azure/storage-blob": "^12.18.0",
     "@blazity/nest-file-fastify": "^1.0.0",
+    "@dqbd/tiktoken": "^1.0.21",
     "@fastify/helmet": "^11.0.0",
     "@fastify/multipart": "^8.0.0",
     "@fastify/rate-limit": "^8.0.3",
@@ -73,6 +74,7 @@
     "class-transformer": "0.5.1",
     "class-transformer-validator": "^0.9.1",
     "class-validator": "^0.14.0",
+    "crypto-js": "^4.2.0",
     "curlconverter": "^4.9.0",
     "dotenv": "16.0.1",
     "fastify": "4.28.1",
@@ -91,6 +93,7 @@
     "passport": "0.6.0",
     "passport-google-oauth20": "^2.0.0",
     "passport-jwt": "^4.0.1",
+    "pdf-parse": "^1.1.1",
     "pino": "^8.15.3",
     "pino-pretty": "^10.2.0",
     "prom-client": "^15.1.3",
@@ -109,13 +112,15 @@
     "@jest/types": "^29.6.3",
     "@nestjs/testing": "^10.1.3",
     "@types/chai": "^4.3.5",
+    "@types/crypto-js": "^4.2.2",
     "@types/gravatar": "1.8.3",
     "@types/jest": "^29.5.3",
     "@types/js-yaml": "^4.0.7",
     "@types/jsonwebtoken": "^9.0.9",
     "@types/mime-types": "^3.0.1",
     "@types/node": "16.11.56",
     "@types/passport-jwt": "^3.0.9",
+    "@types/pdf-parse": "^1.1.5",
     "@types/supertest": "^2.0.12",
     "@types/uuid": "^9.0.5",
     "@types/ws": "^8.5.13",
@@ -140,6 +145,6 @@
   },
   "packageManager": "[email protected]",
   "optionalDependencies": {
-    "@sparrowapp-dev/stripe-billing": "^1.1.1"
+    "@sparrowapp-dev/stripe-billing": "^1.1.7"
   }
 }