llm security observability (#235)

llmguard/langfuse-sec.ipynb

@@ -0,0 +1,207 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from dotenv import load_dotenv\n",
+    "load_dotenv()\n",
+    "import os"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langfuse import Langfuse\n",
+    "\n",
+    "langfuse = Langfuse(\n",
+    "  secret_key=os.getenv(\"LANGFUSE_SECRET_KEY\"),\n",
+    "  public_key=os.getenv(\"LANGFUSE_PUBLIC_KEY\"),\n",
+    "  host=os.getenv(\"LANGFUSE_HOST\")\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from llm_guard.input_scanners import Anonymize\n",
+    "from llm_guard.input_scanners.anonymize_helpers import BERT_LARGE_NER_CONF\n",
+    "from langfuse.openai import openai # OpenAI integration\n",
+    "from langfuse.decorators import observe, langfuse_context\n",
+    "from llm_guard.output_scanners import Deanonymize\n",
+    "from llm_guard.vault import Vault"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "vault = Vault()\n",
+    " \n",
+    "@observe()\n",
+    "def anonymize(input: str):\n",
+    "  scanner = Anonymize(vault, preamble=\"Insert before prompt\", allowed_names=[\"John Doe\"], hidden_names=[\"Test LLC\"],\n",
+    "                    recognizer_conf=BERT_LARGE_NER_CONF, language=\"en\")\n",
+    "  sanitized_prompt, is_valid, risk_score = scanner.scan(prompt)\n",
+    "  return sanitized_prompt\n",
+    " \n",
+    "@observe()\n",
+    "def deanonymize(sanitized_prompt: str, answer: str):\n",
+    "  scanner = Deanonymize(vault)\n",
+    "  sanitized_model_output, is_valid, risk_score = scanner.scan(sanitized_prompt, answer)\n",
+    " \n",
+    "  return sanitized_model_output"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "@observe()\n",
+    "def summarize_transcript(prompt: str):\n",
+    "  sanitized_prompt = anonymize(prompt)\n",
+    " \n",
+    "  answer = openai.chat.completions.create(\n",
+    "        model=\"gpt-3.5-turbo\",\n",
+    "        max_tokens=100,\n",
+    "        messages=[\n",
+    "          {\"role\": \"system\", \"content\": \"Summarize the given court transcript.\"},\n",
+    "          {\"role\": \"user\", \"content\": sanitized_prompt}\n",
+    "        ],\n",
+    "    ).choices[0].message.content\n",
+    " \n",
+    "  sanitized_model_output = deanonymize(sanitized_prompt, answer)\n",
+    " \n",
+    "  return sanitized_model_output"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[2m2024-12-11 12:31:49\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mNo entity types provided, using default\u001b[0m \u001b[36mdefault_entities\u001b[0m=\u001b[35m['CREDIT_CARD', 'CRYPTO', 'EMAIL_ADDRESS', 'IBAN_CODE', 'IP_ADDRESS', 'PERSON', 'PHONE_NUMBER', 'US_SSN', 'US_BANK_NUMBER', 'CREDIT_CARD_RE', 'UUID', 'EMAIL_ADDRESS_RE', 'US_SSN_RE']\u001b[0m\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of the model checkpoint at dslim/bert-large-NER were not used when initializing BertForTokenClassification: ['bert.pooler.dense.bias', 'bert.pooler.dense.weight']\n",
+      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\u001b[2m2024-12-11 12:31:50\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mInitialized NER model         \u001b[0m \u001b[36mdevice\u001b[0m=\u001b[35mdevice(type='mps')\u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mModel(path='dslim/bert-large-NER', subfolder='', revision='13e784dccceca07aee7a7aab4ad487c605975423', onnx_path='dslim/bert-large-NER', onnx_revision='13e784dccceca07aee7a7aab4ad487c605975423', onnx_subfolder='onnx', onnx_filename='model.onnx', kwargs={}, pipeline_kwargs={'batch_size': 1, 'device': device(type='mps'), 'aggregation_strategy': 'simple', 'ignore_labels': ['O', 'CARDINAL']}, tokenizer_kwargs={'model_input_names': ['input_ids', 'attention_mask']})\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:50\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLoaded regex pattern          \u001b[0m \u001b[36mgroup_name\u001b[0m=\u001b[35mCREDIT_CARD_RE\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:50\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLoaded regex pattern          \u001b[0m \u001b[36mgroup_name\u001b[0m=\u001b[35mUUID\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:50\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLoaded regex pattern          \u001b[0m \u001b[36mgroup_name\u001b[0m=\u001b[35mEMAIL_ADDRESS_RE\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:50\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLoaded regex pattern          \u001b[0m \u001b[36mgroup_name\u001b[0m=\u001b[35mUS_SSN_RE\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:50\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLoaded regex pattern          \u001b[0m \u001b[36mgroup_name\u001b[0m=\u001b[35mBTC_ADDRESS\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:50\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLoaded regex pattern          \u001b[0m \u001b[36mgroup_name\u001b[0m=\u001b[35mURL_RE\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:50\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLoaded regex pattern          \u001b[0m \u001b[36mgroup_name\u001b[0m=\u001b[35mCREDIT_CARD\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:50\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLoaded regex pattern          \u001b[0m \u001b[36mgroup_name\u001b[0m=\u001b[35mEMAIL_ADDRESS_RE\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:50\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLoaded regex pattern          \u001b[0m \u001b[36mgroup_name\u001b[0m=\u001b[35mPHONE_NUMBER_ZH\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:50\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLoaded regex pattern          \u001b[0m \u001b[36mgroup_name\u001b[0m=\u001b[35mPHONE_NUMBER_WITH_EXT\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:50\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLoaded regex pattern          \u001b[0m \u001b[36mgroup_name\u001b[0m=\u001b[35mDATE_RE\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:50\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLoaded regex pattern          \u001b[0m \u001b[36mgroup_name\u001b[0m=\u001b[35mTIME_RE\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:50\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLoaded regex pattern          \u001b[0m \u001b[36mgroup_name\u001b[0m=\u001b[35mHEX_COLOR\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:50\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLoaded regex pattern          \u001b[0m \u001b[36mgroup_name\u001b[0m=\u001b[35mPRICE_RE\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:50\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLoaded regex pattern          \u001b[0m \u001b[36mgroup_name\u001b[0m=\u001b[35mPO_BOX_RE\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:51\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1msplitting the text into chunks\u001b[0m \u001b[36mlength\u001b[0m=\u001b[35m810\u001b[0m \u001b[36mmodel_max_length\u001b[0m=\u001b[35m512\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:51\u001b[0m [\u001b[33m\u001b[1mwarning  \u001b[0m] \u001b[1mFound sensitive data in the prompt and replaced it\u001b[0m \u001b[36mmerged_results\u001b[0m=\u001b[35m[type: PERSON, start: 8, end: 13, score: 0.8100000023841858, type: PERSON, start: 91, end: 96, score: 0.7300000190734863, type: PERSON, start: 203, end: 214, score: 1.0]\u001b[0m \u001b[36mrisk_score\u001b[0m=\u001b[35m1.0\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:52\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mReplaced placeholder with real value\u001b[0m \u001b[36mplaceholder\u001b[0m=\u001b[35m[REDACTED_PERSON_2]\u001b[0m\n",
+      "\u001b[2m2024-12-11 12:31:52\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mReplaced placeholder with real value\u001b[0m \u001b[36mplaceholder\u001b[0m=\u001b[35m[REDACTED_PERSON_1]\u001b[0m\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "'Ms. Hyman, a lawyer with over 19 years of experience in mass torts and class actions, including cases against tech giants like Facebook and Google, is about to testify before a committee. She has represented clients in various legal actions, such as those related to data breaches, privacy violations, and mass tort litigation involving medical devices.'"
+      ]
+     },
+     "execution_count": 15,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "prompt = \"So, Ms. Hyman, you should feel free to turn your video on and commence your testimony. Ms. Hyman: Thank you, Your Honor. Good morning. Thank you for the opportunity to address this Committee. My name is Kelly Hyman and I am the founder and managing partner of the Hyman Law Firm, P.A. I’ve been licensed to practice law over 19 years, with the last 10 years focusing on representing plaintiffs in mass torts and class actions. I have represented clients in regards to class actions involving data breaches and privacy violations against some of the largest tech companies, including Facebook, Inc., and Google, LLC. Additionally, I have represented clients in mass tort litigation, hundreds of claimants in individual actions filed in federal court involving ransvaginal mesh and bladder slings. I speak to you\"\n",
+    "\n",
+    "@observe()\n",
+    "def main():\n",
+    "    return summarize_transcript(prompt)\n",
+    "\n",
+    "main()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from langfuse.decorators import observe\n",
+    "from langfuse.openai import openai # OpenAI integration\n",
+    " \n",
+    "@observe()\n",
+    "def story():\n",
+    "    return openai.chat.completions.create(\n",
+    "        model=\"gpt-3.5-turbo\",\n",
+    "        max_tokens=100,\n",
+    "        messages=[\n",
+    "          {\"role\": \"system\", \"content\": \"You are a great storyteller.\"},\n",
+    "          {\"role\": \"user\", \"content\": \"Once upon a time in a galaxy far, far away...\"}\n",
+    "        ],\n",
+    "    ).choices[0].message.content\n",
+    " \n",
+    "@observe()\n",
+    "def main():\n",
+    "    return story()\n",
+    " \n",
+    "main()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "bedrock",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}