TLM-PII-Detection/TLM-PII-Detection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "9366c5dc-ca5d-43f9-8e76-782081950b28",
   "metadata": {},
   "source": [
    "## Imports and Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79992c21",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set your API key\n",
    "import os\n",
    "os.environ[\"CLEANLAB_TLM_API_KEY\"] = \"<API key>\"  # Get your API key from: https://tlm.cleanlab.ai/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c9b2e88b-ddf8-409b-8c9e-c38b9bd46a32",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "pd.options.mode.chained_assignment = None\n",
    "\n",
    "from cleanlab_tlm import TLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "23f802e0-d226-4f4c-9c4f-bf73e704340d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Customer: Hi, my name is John Doe. I need help...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Customer: Hello, I want to update my credit ca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Agent: I see you want to reset your account. C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Customer: I received a charge on my card endin...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Customer: My phone numbers 555-234-5678 and 55...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text\n",
       "0  Customer: Hi, my name is John Doe. I need help...\n",
       "1  Customer: Hello, I want to update my credit ca...\n",
       "2  Agent: I see you want to reset your account. C...\n",
       "3  Customer: I received a charge on my card endin...\n",
       "4  Customer: My phone numbers 555-234-5678 and 55..."
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Helper method to parse input into a json object\n",
    "def parse_to_json(raw_input):\n",
    "    # Check if already valid JSON object\n",
    "    if type(raw_input) == dict:\n",
    "        return raw_input\n",
    "    # Find the start of the JSON content\n",
    "    start_index = raw_input.find('```json')\n",
    "    if start_index == -1:\n",
    "        start_index = 0 - len('```json')\n",
    "    start_index += len('```json')\n",
    "    # Find the end of the JSON content\n",
    "    end_index = raw_input.rfind('```')\n",
    "    if end_index == -1:\n",
    "        end_index = len(raw_input)\n",
    "    # Extract the JSON content\n",
    "    json_content = raw_input[start_index:end_index].strip()\n",
    "    # Parse the JSON content\n",
    "    try:\n",
    "        data = json.loads(json_content)\n",
    "    except json.JSONDecodeError as e:\n",
    "        return dict()\n",
    "    return data\n",
    "\n",
    "# Returns list of all values in json\n",
    "def get_all_values(json_object):\n",
    "    return [item for sublist in json_object.values() for item in (sublist if isinstance(sublist, list) else [sublist])]\n",
    "    \n",
    "# Check if text contains all of the pii_items\n",
    "def check_all_values(text, pii_items):\n",
    "    if len(pii_items) == 0:\n",
    "        return True\n",
    "    return all([pii_item in text for pii_item in pii_items])\n",
    "\n",
    "# List of PII items\n",
    "pii_items = {\n",
    "    \"social security numbers (SSNs)\": {\n",
    "        \"description\": \"A unique identifier of 9 digits (sometimes separated by dashes in the form ###-##-#### or #########) assigned to U.S. citizens and some residents for tax and social benefits purposes\",\n",
    "        \"key\": 'ssns'\n",
    "    },\n",
    "    \"credit card numbers\": {\n",
    "        \"description\": \"A numeric string (16 digits, sometimes grouped in 4 sets of 4 digits) that identifies a credit card used for financial transactions\",\n",
    "        \"key\": 'credit_cards'\n",
    "    },\n",
    "    \"names\": {\n",
    "        \"description\": \"A person's first and last name, excluding instances where the name is found within email addresses\",\n",
    "        \"key\": 'names'\n",
    "    },\n",
    "    \"email addresses\": {\n",
    "        \"description\": \"An identifier for electronic mail communication that is usually in the format example@domain.com\",\n",
    "        \"key\": 'emails'\n",
    "    },\n",
    "    \"phone numbers\": {\n",
    "        \"description\": \"A sequence of 7, 10, or 11 digits used for communication via mobile or landline. Some formats include #######, ########## , (###) ###-#### , ###-###-#### , +1 (###) ###-#### , +1 ###-###-####\",\n",
    "        \"key\": 'phone_numbers'\n",
    "    },\n",
    "    \"API keys\": {\n",
    "        \"description\": \"A unique token used to authenticate requests to an API service, can be alphanumeric strings, typically 20-40 characters long\",\n",
    "        \"key\": 'api_keys'\n",
    "    }\n",
    "}\n",
    "\n",
    "# Generate fake PII data in the context of customer service chat interactions\n",
    "data = [\n",
    "    {\"text\": \"Customer: Hi, my name is John Doe. I need help with my account. Can you check it using my email john.doe@example.com and my phone number 555-123-4567?\"},\n",
    "    {\"text\": \"Customer: Hello, I want to update my credit card information. My old card is 4111111111111112 and my current card number is 4111111111111111.\"},\n",
    "    {\"text\": \"Agent: I see you want to reset your account. Can you confirm your SSN? Customer: Sure, it's 987-65-4321.\"},\n",
    "    {\"text\": \"Customer: I received a charge on my card ending in 1234. Can you check why this happened? My email is jane.smith@example.net.\"},\n",
    "    {\"text\": \"Customer: My phone numbers 555-234-5678 and 555-234-5677 aren’t associated with my account anymore. Can you update it with jane.doe@domain.com?\"},\n",
    "    {\"text\": \"Agent: For security purposes, we cannot disclose API keys over chat. Customer: Here is mine: sk_test_51HfVxECq4sTBwnjJd123T34ef7.\"},\n",
    "    {\"text\": \"Customer: I’m concerned because my husband and I's ssns 456-78-9123 and 716-72-4618 were submitted on your site, but I didn't get confirmation.\"},\n",
    "    {\"text\": \"Customer: I think someone stole my information. My email is mark.stevens@example.com, and I got suspicious activity on card 4000 0000 0000 0002.\"},\n",
    "    {\"text\": \"Customer: Here is all my info: 555-23-5677, 4111111111111112, Alex Jones, alex.jones@example.org, 555-789-1234, cl_2sdDfjWErLfjweTso392Dd\"},\n",
    "    {\"text\": \"Customer: My API key for service is sk_live_2sdDfjWErLfjweTso392Dd, and I am facing issues while using it.\"},\n",
    "    {\"text\": \"Customer: Hi! I’m having some issues accessing my account. Could you assist me? Agent: Hello! Sure, I'd be happy to help. Could you describe the issue you're experiencing? Customer: Every time I try to log in, I get an error message saying, “Unable to authenticate.” I’m sure I’m entering the right password. Agent: Got it. That error typically appears if there’s a password issue or if multi-factor authentication needs updating. Have you tried resetting your password? Customer: Not yet. If I reset it, will I lose any saved data? Agent: Not at all! Resetting your password only changes your login credentials; all your saved data will remain intact. Customer: Perfect! Can you guide me through the password reset process? Agent: Absolutely. First, head to our login page and click on “Forgot Password.” Then, follow the prompts to enter the email associated with your account. You’ll receive a reset link in your email.\"},\n",
    "    {\"text\": \"Customer: Hi, I need help with an order issue. I ordered something on 09/15/2023, and I still haven’t received it. My order number is 2048573. Agent: I’m here to help! Let me check the status of order #2048573 for you. One moment please. Customer: Sure, thanks! Agent: Thank you for waiting. It looks like your order was shipped on 09/18/2023 and is currently in transit. You should receive it by 09/25/2023. Customer: Got it! Will I be able to track it? Agent: Yes, you can! Your tracking number is 1Z999AA10123456784. Just go to our tracking page, enter the number, and you’ll see the latest status. Customer: Great, thank you. One more thing—I saw a charge of $19.99 on my account. Could you explain what it’s for? Agent: Sure thing. That charge on 09/20/2023 was for a subscription renewal. Let me know if you'd like to update your subscription settings. Customer: I’d like to change it to the monthly plan instead of the annual one. Agent: No problem! I’ll switch your subscription to the monthly plan, which is $9.99 per month, starting with your next billing cycle.\"},\n",
    "    {\"text\": \"Customer: Hi, I have a question about canceling my service. Agent: Hello! I can assist you with that. Just to confirm, are you looking to cancel immediately or at the end of the current billing period? Customer: At the end of the period would be good. Agent: Understood! Before I proceed, I need to read a detailed disclaimer regarding our cancellation policy. Here it is: 'By proceeding with the cancellation of your service, you agree to and acknowledge the following terms: (1) All account data, including but not limited to settings, preferences, saved information, and any content generated through our platform, will be permanently deleted 30 days after the end of your current billing period. It is your responsibility to back up any important data before this date. (2) All pending charges for services rendered up to the cancellation date will remain due and payable, and failure to settle these charges may result in additional fees. (3) Any active warranties or service guarantees that were applicable under your account will become null and void upon the effective cancellation date. (4) By canceling, you forfeit access to all promotional benefits, loyalty rewards, or other incentives previously applied to your account, and any accumulated points or credits will also be forfeited. (5) Our team will no longer be able to provide support for retrieving data, reactivating prior settings, or addressing issues related to the service post-cancellation. (6) Re-enrollment following cancellation will be treated as a new account registration, and any prior status, discounts, or service history may not be reinstated. Please review the full policy on our website for complete details. By choosing to continue, you affirm that you have read, understood, and agree to these terms in their entirety.' Customer: That’s really thorough! I understand the terms. Let’s go ahead with the cancellation. Agent: Thank you for confirming. I’ll schedule your cancellation for the end of your current billing period. Please let us know if there’s anything else we can assist with before then!\"}\n",
    "]\n",
    "\n",
    "# Convert to DataFrame\n",
    "df_data = pd.DataFrame(data)\n",
    "df_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "288c1535-df6e-4d01-aaa1-4a942418e5bd",
   "metadata": {},
   "source": [
    "## Use TLM to indentify PII items"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "fb7c7910-7bba-4b49-9298-d7023b08a661",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize TLM\n",
    "tlm = TLM(options={\"log\": [\"explanation\"]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e6bcf835-57ed-4cd3-89c9-79073e952572",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Prompt to extract PII from provided text\n",
    "pii_prompt = '''You are an expert compliance officer. Your task is to scrupulously read through \n",
    "the following text and extract pieces of Personally Identifiable Information (PII). The specific PII items\n",
    "you are looking for are the following: social security numbers (SSNs), credit card numbers,\n",
    "names(first and last that are not within email addresses), email addresses, phone numbers, and API keys.\n",
    "You are only allowed to respond with PII that is explicitly contained in the text in the correct form of the type of PII.\n",
    "You may not infer PII from other PII items. If you find names within email addresses,\n",
    "please only include the email addresses themselves as PII items, do not include the names within the email addresses. \n",
    "Please format your response as a json object with keys: [ssns, credit_cards, names, emails, phone_numbers, api_keys] and values of\n",
    "lists that you fill with found PII items. If there are no items for a particular PII item please\n",
    "leave the list empty.\n",
    "Here is the text: {}\n",
    "'''\n",
    "\n",
    "# Get list of all text examples\n",
    "text_examples = df_data.text.values.tolist()\n",
    "\n",
    "# Generate list of prompts so we can run them all at the same time\n",
    "pii_prompts = [pii_prompt.format(text_example) for text_example in text_examples]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "12eb77bf-6e65-4ed4-9665-3003189e5e14",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Querying TLM... 100%|█████████████████████████████████████████████████████|\n"
     ]
    }
   ],
   "source": [
    "# Query TLM\n",
    "responses = tlm.prompt(pii_prompts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "64fcedd8-e3b8-4835-9f95-db850660020d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>pii_items</th>\n",
       "      <th>pii_items_score</th>\n",
       "      <th>pii_items_expl</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Customer: Hi, my name is John Doe. I need help...</td>\n",
       "      <td>```json\\n{\\n  \"ssns\": [],\\n  \"credit_cards\": [...</td>\n",
       "      <td>0.972159</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Customer: Hello, I want to update my credit ca...</td>\n",
       "      <td>```json\\n{\\n  \"ssns\": [],\\n  \"credit_cards\": [...</td>\n",
       "      <td>0.953389</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Agent: I see you want to reset your account. C...</td>\n",
       "      <td>```json\\n{\\n  \"ssns\": [\"987-65-4321\"],\\n  \"cre...</td>\n",
       "      <td>0.945038</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  \\\n",
       "0  Customer: Hi, my name is John Doe. I need help...   \n",
       "1  Customer: Hello, I want to update my credit ca...   \n",
       "2  Agent: I see you want to reset your account. C...   \n",
       "\n",
       "                                           pii_items  pii_items_score  \\\n",
       "0  ```json\\n{\\n  \"ssns\": [],\\n  \"credit_cards\": [...         0.972159   \n",
       "1  ```json\\n{\\n  \"ssns\": [],\\n  \"credit_cards\": [...         0.953389   \n",
       "2  ```json\\n{\\n  \"ssns\": [\"987-65-4321\"],\\n  \"cre...         0.945038   \n",
       "\n",
       "                                    pii_items_expl  \n",
       "0  Did not find a reason to doubt trustworthiness.  \n",
       "1  Did not find a reason to doubt trustworthiness.  \n",
       "2  Did not find a reason to doubt trustworthiness.  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Add results to df\n",
    "df_data[\"pii_items\"] = [entry['response'] for entry in  responses]\n",
    "df_data[\"pii_items_score\"] = [entry['trustworthiness_score'] for entry in  responses]\n",
    "df_data[\"pii_items_expl\"] = [entry['log']['explanation'] for entry in  responses]\n",
    "df_data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "94518fa0-25b6-4cd9-b84d-5da1a1c2268d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Customer: Hi, my name is John Doe. I need help with my account. Can you check it using my email john.doe@example.com and my phone number 555-123-4567?\n",
      "0.9721594461113267\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'ssns': [],\n",
       " 'credit_cards': [],\n",
       " 'names': ['John Doe'],\n",
       " 'emails': ['john.doe@example.com'],\n",
       " 'phone_numbers': ['555-123-4567'],\n",
       " 'api_keys': []}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "-------------------------------------------\n",
      "\n",
      "Customer: Hello, I want to update my credit card information. My old card is 4111111111111112 and my current card number is 4111111111111111.\n",
      "0.9533886976036989\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'ssns': [],\n",
       " 'credit_cards': ['4111111111111112', '4111111111111111'],\n",
       " 'names': [],\n",
       " 'emails': [],\n",
       " 'phone_numbers': [],\n",
       " 'api_keys': []}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "-------------------------------------------\n",
      "\n",
      "Agent: I see you want to reset your account. Can you confirm your SSN? Customer: Sure, it's 987-65-4321.\n",
      "0.9450384617795651\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'ssns': ['987-65-4321'],\n",
       " 'credit_cards': [],\n",
       " 'names': [],\n",
       " 'emails': [],\n",
       " 'phone_numbers': [],\n",
       " 'api_keys': []}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "-------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# View some results\n",
    "for idx, row in df_data.head(3).iterrows():\n",
    "    print(row.text)\n",
    "    print(row.pii_items_score)\n",
    "    print()\n",
    "    display(parse_to_json(row.pii_items))\n",
    "    print()\n",
    "    print(\"-------------------------------------------\")\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b68d45fe-f523-4257-95ed-cf12f7a0bf90",
   "metadata": {},
   "source": [
    "## Handling Scenarios w/ Human-in-the-Loop\n",
    "With a human-in-the-loop, we can focus on examples that have a low trustworthiness score and you can decide how to best handle them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9d664df9-689e-42f2-afc5-4fea9505a51c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>pii_items</th>\n",
       "      <th>pii_items_score</th>\n",
       "      <th>pii_items_expl</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Customer: My phone numbers 555-234-5678 and 55...</td>\n",
       "      <td>```json\\n{\\n  \"ssns\": [],\\n  \"credit_cards\": [...</td>\n",
       "      <td>0.934779</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Customer: I received a charge on my card endin...</td>\n",
       "      <td>```json\\n{\\n  \"ssns\": [],\\n  \"credit_cards\": [...</td>\n",
       "      <td>0.935201</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Agent: I see you want to reset your account. C...</td>\n",
       "      <td>```json\\n{\\n  \"ssns\": [\"987-65-4321\"],\\n  \"cre...</td>\n",
       "      <td>0.945038</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Customer: Hi, I need help with an order issue....</td>\n",
       "      <td>```json\\n{\\n  \"ssns\": [],\\n  \"credit_cards\": [...</td>\n",
       "      <td>0.947829</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Customer: Hi! I’m having some issues accessing...</td>\n",
       "      <td>```json\\n{\\n  \"ssns\": [],\\n  \"credit_cards\": [...</td>\n",
       "      <td>0.947878</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text  \\\n",
       "4   Customer: My phone numbers 555-234-5678 and 55...   \n",
       "3   Customer: I received a charge on my card endin...   \n",
       "2   Agent: I see you want to reset your account. C...   \n",
       "11  Customer: Hi, I need help with an order issue....   \n",
       "10  Customer: Hi! I’m having some issues accessing...   \n",
       "\n",
       "                                            pii_items  pii_items_score  \\\n",
       "4   ```json\\n{\\n  \"ssns\": [],\\n  \"credit_cards\": [...         0.934779   \n",
       "3   ```json\\n{\\n  \"ssns\": [],\\n  \"credit_cards\": [...         0.935201   \n",
       "2   ```json\\n{\\n  \"ssns\": [\"987-65-4321\"],\\n  \"cre...         0.945038   \n",
       "11  ```json\\n{\\n  \"ssns\": [],\\n  \"credit_cards\": [...         0.947829   \n",
       "10  ```json\\n{\\n  \"ssns\": [],\\n  \"credit_cards\": [...         0.947878   \n",
       "\n",
       "                                     pii_items_expl  \n",
       "4   Did not find a reason to doubt trustworthiness.  \n",
       "3   Did not find a reason to doubt trustworthiness.  \n",
       "2   Did not find a reason to doubt trustworthiness.  \n",
       "11  Did not find a reason to doubt trustworthiness.  \n",
       "10  Did not find a reason to doubt trustworthiness.  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_data.sort_values(by=\"pii_items_score\", ascending=True).head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "886da994-6d61-4f82-8851-246fceec0c44",
   "metadata": {},
   "source": [
    "## Handling Scenarios Automatically w/o Human-in-the-Loop\n",
    "We can handle the following scenarios automatically:\n",
    "- TLM score falls below fixed threshold (i.e. 0.8)\n",
    "- JSON parsing the original LLM output fails\n",
    "- One of the JSON values is not present in the original document text\n",
    "\n",
    "The strategy will be:\n",
    "1. Filter the dataframe to include only examples of the scenarios listed above\n",
    "2. Prompt TLM to identify examples where any PII exists (binary Yes/No)\n",
    "   - A) If NO with high TLM confidence, return empty JSON\n",
    "   - B) Otherwise, we prompt the model to audit for each of the PII items separately (which is more trustworthy than auditing for all types of PII simultaneously), and then construct the resulting JSON based on these individual analyses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f76ce231-1e02-4c2d-a2ed-cf92a57a0317",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>pii_items</th>\n",
       "      <th>pii_items_score</th>\n",
       "      <th>pii_items_expl</th>\n",
       "      <th>low_tlm_score</th>\n",
       "      <th>pii_items_json</th>\n",
       "      <th>pii_items_list</th>\n",
       "      <th>pii_items_valid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Agent: I see you want to deactivate your accou...</td>\n",
       "      <td>```json\\n{\\n  \"ssns\": [\"123-45-5678\"],\\n  \"cre...</td>\n",
       "      <td>0.7</td>\n",
       "      <td>Explanation string.</td>\n",
       "      <td>True</td>\n",
       "      <td>{'ssns': ['123-45-5678'], 'credit_cards': [], ...</td>\n",
       "      <td>[123-45-5678]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Agent: Hello! What is your issue? Customer: I ...</td>\n",
       "      <td>JSON{None}</td>\n",
       "      <td>0.8</td>\n",
       "      <td>Explanation string.</td>\n",
       "      <td>False</td>\n",
       "      <td>{}</td>\n",
       "      <td>[]</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Agent: I see you want to check the status of a...</td>\n",
       "      <td>```json\\n{\\n  \"ssns\": [],\\n  \"credit_cards\": [...</td>\n",
       "      <td>0.8</td>\n",
       "      <td>Explanation string.</td>\n",
       "      <td>False</td>\n",
       "      <td>{'ssns': [], 'credit_cards': [], 'names': [], ...</td>\n",
       "      <td>[11234567890]</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text  \\\n",
       "13  Agent: I see you want to deactivate your accou...   \n",
       "14  Agent: Hello! What is your issue? Customer: I ...   \n",
       "15  Agent: I see you want to check the status of a...   \n",
       "\n",
       "                                            pii_items  pii_items_score  \\\n",
       "13  ```json\\n{\\n  \"ssns\": [\"123-45-5678\"],\\n  \"cre...              0.7   \n",
       "14                                         JSON{None}              0.8   \n",
       "15  ```json\\n{\\n  \"ssns\": [],\\n  \"credit_cards\": [...              0.8   \n",
       "\n",
       "         pii_items_expl  low_tlm_score  \\\n",
       "13  Explanation string.           True   \n",
       "14  Explanation string.          False   \n",
       "15  Explanation string.          False   \n",
       "\n",
       "                                       pii_items_json pii_items_list  \\\n",
       "13  {'ssns': ['123-45-5678'], 'credit_cards': [], ...  [123-45-5678]   \n",
       "14                                                 {}             []   \n",
       "15  {'ssns': [], 'credit_cards': [], 'names': [], ...  [11234567890]   \n",
       "\n",
       "    pii_items_valid  \n",
       "13             True  \n",
       "14             True  \n",
       "15            False  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Here we add one example of each scenario manually for purposes of this demonstration\n",
    "example_data = [\n",
    "        # TLM Score falls below threshold\n",
    "        [\"Agent: I see you want to deactivate your account. Can you confirm your number? Customer: Sure, it's 123-45-5678\", '```json\\n{\\n  \"ssns\": [\"123-45-5678\"],\\n  \"credit_cards\": [],\\n  \"names\": [],\\n  \"emails\": [],\\n  \"phone_numbers\": [],\\n  \"api_keys\": []\\n}\\n```', 0.7, \"Explanation string.\"],\n",
    "        # JSON parsing fails (invalid json)\n",
    "        ['Agent: Hello! What is your issue? Customer: I need help with my account.', \"JSON{None}\", 0.8, \"Explanation string.\"],\n",
    "        # One of JSON values not found\n",
    "        [\"Agent: I see you want to check the status of account. Can you confirm your phone number? Customer: Sure, it's +1 (123) 456-7890\", '```json\\n{\\n  \"ssns\": [],\\n  \"credit_cards\": [],\\n  \"names\": [],\\n  \"emails\": [],\\n  \"phone_numbers\": [\"11234567890\"],\\n  \"api_keys\": []\\n}\\n```', 0.8, \"Explanation string.\"],\n",
    "        ]\n",
    "example_df = pd.DataFrame(example_data, columns = [\"text\", \"pii_items\", \"pii_items_score\", \"pii_items_expl\"]).reset_index(drop=True)\n",
    "df = pd.concat([df_data, example_df]).drop_duplicates().reset_index(drop=True)\n",
    "\n",
    "# Add in columns to gelp filter to the above scenarios\n",
    "# Check which responses have a low trustworthiness score\n",
    "df[\"low_tlm_score\"] = df[\"pii_items_score\"].apply(lambda score: True if score <0.8 else False)\n",
    "# Parse the raw LLM response to a JSON object, those that fail will be an empty dict()\n",
    "df[\"pii_items_json\"] = df[\"pii_items\"].apply(parse_to_json)\n",
    "# Create list of all pii items \n",
    "df[\"pii_items_list\"] = df[\"pii_items_json\"].apply(lambda json_object: get_all_values(json_object))\n",
    "# Check that all items are contained within original text\n",
    "df[\"pii_items_valid\"] = df.apply(lambda row: check_all_values(row['text'], row['pii_items_list']), axis=1)\n",
    "\n",
    "# Here we save a version to modify directly when we make changes automatically\n",
    "df_original = df.copy()\n",
    "df_original.tail(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8bb6cfc1-1b09-4acc-9e3b-6e47d5bcad60",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Querying TLM... 100%|█████████████████████████████████████████████████████|\n",
      "Querying TLM... 100%|█████████████████████████████████████████████████████|\n",
      "Querying TLM... 100%|█████████████████████████████████████████████████████|\n"
     ]
    }
   ],
   "source": [
    "# Filter to subset of columns that we need\n",
    "df_subset = df[[\"text\", \"pii_items_json\", \"low_tlm_score\", \"pii_items_valid\"]]\n",
    "# Filter to include above flagged scenarios\n",
    "df_flagged = df_subset[\n",
    "    (~df_subset.pii_items_valid) | \n",
    "    (df_subset.pii_items_json.apply(lambda x: isinstance(x, dict) and len(x) == 0)) | \n",
    "    df_subset.low_tlm_score\n",
    "]\n",
    "\n",
    "# Step 1: Prompt TLM to determine if there is PII, binary YES/NO \n",
    "# In this workflow we are expending more compute to get higher confidence answers for examples that we flagged above.\n",
    "\n",
    "# We first prompt TLM to determine if PII exists, yes or no, so that we can return an empty json object for examples we confidently know do not contain PII.\n",
    "pii_prompt_binary = '''You are an expert compliance officer. Your task is to determine if the following text contains any of the following\n",
    "Personally Identifiable Information (PII) items: social security numbers (SSNs), credit card numbers,\n",
    "names, email addresses, phone numbers, and API keys.\n",
    "Respond with only 'yes' or 'no' with no leading or trailing text.\n",
    "Here is the text: {}\n",
    "'''\n",
    "\n",
    "# Get list of all text examples\n",
    "text_examples = df_flagged.text.values.tolist()\n",
    "\n",
    "# Generate list of prompts so we can run them all at the same time\n",
    "pii_prompts_binary = [pii_prompt_binary.format(text_example) for text_example in text_examples]\n",
    "\n",
    "# Query TLM\n",
    "binary_responses = tlm.prompt(pii_prompts_binary)\n",
    "\n",
    "# Add results to df\n",
    "df_flagged[\"has_pii\"] = [entry['response'] for entry in  binary_responses]\n",
    "df_flagged[\"has_pii_score\"] = [entry['trustworthiness_score'] for entry in binary_responses]\n",
    "df_flagged[\"has_pii_expl\"] = [entry['log']['explanation'] for entry in binary_responses]\n",
    "df_flagged.head()\n",
    "\n",
    "# Step 2: If there is confidently no PII, insert in empty json\n",
    "\n",
    "# Define empty JSON structure\n",
    "empty_json = {'ssns': [],\n",
    " 'credit_cards': [],\n",
    " 'names': [],\n",
    " 'emails': [],\n",
    " 'phone_numbers': [],\n",
    " 'api_keys': []}\n",
    "\n",
    "# Find examples that confidently don't have pii\n",
    "has_no_pii_threshold = 0.8\n",
    "df_no_pii_high_trust = df_flagged[(df_flagged.has_pii==\"no\") & (df_flagged.has_pii_score > has_no_pii_threshold)]\n",
    "\n",
    "# Iterate through df_original and insert empty json \n",
    "for idx, row in df_no_pii_high_trust.iterrows():\n",
    "    df_original.at[idx, \"pii_items_json\"] = empty_json\n",
    "    df_original.at[idx, \"pii_items_list\"] = []\n",
    "    df_original.at[idx, \"pii_items_score\"] = row.has_pii_score\n",
    "    df_original.at[idx, \"pii_items_expl\"] = row.has_pii_expl\n",
    "\n",
    "# Step 3: For remaining examples, we individually prompt TLM for each PII item and manually construct json\n",
    "\n",
    "# Remove high trust instances we just corrected\n",
    "df_possible_pii = df_flagged.drop(df_no_pii_high_trust.index)\n",
    "\n",
    "# Prompt to extract PII from provided text, now asking for each of the PII items individually\n",
    "pii_prompt_individual = '''You are an expert compliance officer that is trying to find instances of Personally Identifiable Information (PII).\n",
    "Your task is to scrupulously read through the following text and respond with instances of {pii_item} : {pii_item_description}.\n",
    "You are only allowed to respond with instances you find that are explicitly contained in the text.\n",
    "Please respond with all instances of {pii_item} separated by commas.\n",
    "If there are no {pii_item} found, please respond with \"None\".\n",
    "Here is the text: {text}\n",
    "'''\n",
    "\n",
    "# Assemble batches of prompts\n",
    "pii_prompts_individual = []\n",
    "for text in df_possible_pii.text.values:\n",
    "    prompts = []\n",
    "    for pii_item in pii_items:\n",
    "        prompt = pii_prompt_individual.format(pii_item=pii_item,pii_item_description=pii_items[pii_item][\"description\"],text=text)\n",
    "        prompts.append(prompt)\n",
    "    pii_prompts_individual.append(prompts)\n",
    "\n",
    "# Query TLM\n",
    "individual_responses = [tlm.prompt(batch) for batch in pii_prompts_individual]\n",
    "\n",
    "# Compute the minimum trustworthiness score of all individual pii queries\n",
    "min_scores = [min(item['trustworthiness_score'] for item in entry) for entry in individual_responses]\n",
    "\n",
    "# Manually construct JSON from individually responses\n",
    "manual_jsons = []\n",
    "for i in range(len(individual_responses)):\n",
    "    manual_json = dict()\n",
    "    for j, pii_item in enumerate(pii_items):\n",
    "        key = pii_items[pii_item][\"key\"]\n",
    "        response = individual_responses[i][j][\"response\"]\n",
    "        response_list = [x.strip() for x in response.split(\",\")]\n",
    "        # If no PII was found for that item\n",
    "        if response_list[0] == \"None\":\n",
    "            response_list = []\n",
    "        manual_json[key] = response_list\n",
    "    manual_jsons.append(manual_json)\n",
    "\n",
    "# Iterate through df_original and modify rows\n",
    "for i, (idx, row) in enumerate(df_possible_pii.iterrows()):\n",
    "    df_original.at[idx, \"pii_items_json\"] = manual_jsons[i]\n",
    "    df_original.at[idx, \"pii_items_list\"] = get_all_values(manual_jsons[i])\n",
    "    df_original.at[idx, \"pii_items_score\"] = min_scores[i]\n",
    "    df_original.at[idx, \"pii_items_expl\"] = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "e03e2499-4f61-40b9-8f2e-10eda1836e3f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>pii_items_json</th>\n",
       "      <th>pii_items_score</th>\n",
       "      <th>pii_items_expl</th>\n",
       "      <th>pii_items_list</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Customer: Hi, my name is John Doe. I need help...</td>\n",
       "      <td>{'ssns': [], 'credit_cards': [], 'names': ['Jo...</td>\n",
       "      <td>0.972159</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "      <td>[John Doe, john.doe@example.com, 555-123-4567]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Customer: Hello, I want to update my credit ca...</td>\n",
       "      <td>{'ssns': [], 'credit_cards': ['411111111111111...</td>\n",
       "      <td>0.953389</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "      <td>[4111111111111112, 4111111111111111]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Agent: I see you want to reset your account. C...</td>\n",
       "      <td>{'ssns': ['987-65-4321'], 'credit_cards': [], ...</td>\n",
       "      <td>0.945038</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "      <td>[987-65-4321]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Customer: I received a charge on my card endin...</td>\n",
       "      <td>{'ssns': [], 'credit_cards': ['1234'], 'names'...</td>\n",
       "      <td>0.935201</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "      <td>[1234, jane.smith@example.net]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Customer: My phone numbers 555-234-5678 and 55...</td>\n",
       "      <td>{'ssns': [], 'credit_cards': [], 'names': [], ...</td>\n",
       "      <td>0.934779</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "      <td>[jane.doe@domain.com, 555-234-5678, 555-234-5677]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Agent: For security purposes, we cannot disclo...</td>\n",
       "      <td>{'ssns': [], 'credit_cards': [], 'names': [], ...</td>\n",
       "      <td>0.962404</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "      <td>[sk_test_51HfVxECq4sTBwnjJd123T34ef7]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Customer: I’m concerned because my husband and...</td>\n",
       "      <td>{'ssns': ['456-78-9123', '716-72-4618'], 'cred...</td>\n",
       "      <td>0.959646</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "      <td>[456-78-9123, 716-72-4618]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Customer: I think someone stole my information...</td>\n",
       "      <td>{'ssns': [], 'credit_cards': ['4000 0000 0000 ...</td>\n",
       "      <td>0.965663</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "      <td>[4000 0000 0000 0002, mark.stevens@example.com]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Customer: Here is all my info: 555-23-5677, 41...</td>\n",
       "      <td>{'ssns': ['555-23-5677'], 'credit_cards': ['41...</td>\n",
       "      <td>0.958612</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "      <td>[555-23-5677, 4111111111111112, Alex Jones, al...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Customer: My API key for service is sk_live_2s...</td>\n",
       "      <td>{'ssns': [], 'credit_cards': [], 'names': [], ...</td>\n",
       "      <td>0.963837</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "      <td>[sk_live_2sdDfjWErLfjweTso392Dd]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Customer: Hi! I’m having some issues accessing...</td>\n",
       "      <td>{'ssns': [], 'credit_cards': [], 'names': [], ...</td>\n",
       "      <td>0.947878</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Customer: Hi, I need help with an order issue....</td>\n",
       "      <td>{'ssns': [], 'credit_cards': [], 'names': [], ...</td>\n",
       "      <td>0.947829</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Customer: Hi, I have a question about cancelin...</td>\n",
       "      <td>{'ssns': [], 'credit_cards': [], 'names': [], ...</td>\n",
       "      <td>0.957794</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Agent: I see you want to deactivate your accou...</td>\n",
       "      <td>{'ssns': ['123-45-5678'], 'credit_cards': [], ...</td>\n",
       "      <td>0.691028</td>\n",
       "      <td>None</td>\n",
       "      <td>[123-45-5678]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Agent: Hello! What is your issue? Customer: I ...</td>\n",
       "      <td>{'ssns': [], 'credit_cards': [], 'names': [], ...</td>\n",
       "      <td>0.962292</td>\n",
       "      <td>Did not find a reason to doubt trustworthiness.</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Agent: I see you want to check the status of a...</td>\n",
       "      <td>{'ssns': [], 'credit_cards': [], 'names': [], ...</td>\n",
       "      <td>0.988376</td>\n",
       "      <td>None</td>\n",
       "      <td>[+1 (123) 456-7890]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 text  \\\n",
       "0   Customer: Hi, my name is John Doe. I need help...   \n",
       "1   Customer: Hello, I want to update my credit ca...   \n",
       "2   Agent: I see you want to reset your account. C...   \n",
       "3   Customer: I received a charge on my card endin...   \n",
       "4   Customer: My phone numbers 555-234-5678 and 55...   \n",
       "5   Agent: For security purposes, we cannot disclo...   \n",
       "6   Customer: I’m concerned because my husband and...   \n",
       "7   Customer: I think someone stole my information...   \n",
       "8   Customer: Here is all my info: 555-23-5677, 41...   \n",
       "9   Customer: My API key for service is sk_live_2s...   \n",
       "10  Customer: Hi! I’m having some issues accessing...   \n",
       "11  Customer: Hi, I need help with an order issue....   \n",
       "12  Customer: Hi, I have a question about cancelin...   \n",
       "13  Agent: I see you want to deactivate your accou...   \n",
       "14  Agent: Hello! What is your issue? Customer: I ...   \n",
       "15  Agent: I see you want to check the status of a...   \n",
       "\n",
       "                                       pii_items_json  pii_items_score  \\\n",
       "0   {'ssns': [], 'credit_cards': [], 'names': ['Jo...         0.972159   \n",
       "1   {'ssns': [], 'credit_cards': ['411111111111111...         0.953389   \n",
       "2   {'ssns': ['987-65-4321'], 'credit_cards': [], ...         0.945038   \n",
       "3   {'ssns': [], 'credit_cards': ['1234'], 'names'...         0.935201   \n",
       "4   {'ssns': [], 'credit_cards': [], 'names': [], ...         0.934779   \n",
       "5   {'ssns': [], 'credit_cards': [], 'names': [], ...         0.962404   \n",
       "6   {'ssns': ['456-78-9123', '716-72-4618'], 'cred...         0.959646   \n",
       "7   {'ssns': [], 'credit_cards': ['4000 0000 0000 ...         0.965663   \n",
       "8   {'ssns': ['555-23-5677'], 'credit_cards': ['41...         0.958612   \n",
       "9   {'ssns': [], 'credit_cards': [], 'names': [], ...         0.963837   \n",
       "10  {'ssns': [], 'credit_cards': [], 'names': [], ...         0.947878   \n",
       "11  {'ssns': [], 'credit_cards': [], 'names': [], ...         0.947829   \n",
       "12  {'ssns': [], 'credit_cards': [], 'names': [], ...         0.957794   \n",
       "13  {'ssns': ['123-45-5678'], 'credit_cards': [], ...         0.691028   \n",
       "14  {'ssns': [], 'credit_cards': [], 'names': [], ...         0.962292   \n",
       "15  {'ssns': [], 'credit_cards': [], 'names': [], ...         0.988376   \n",
       "\n",
       "                                     pii_items_expl  \\\n",
       "0   Did not find a reason to doubt trustworthiness.   \n",
       "1   Did not find a reason to doubt trustworthiness.   \n",
       "2   Did not find a reason to doubt trustworthiness.   \n",
       "3   Did not find a reason to doubt trustworthiness.   \n",
       "4   Did not find a reason to doubt trustworthiness.   \n",
       "5   Did not find a reason to doubt trustworthiness.   \n",
       "6   Did not find a reason to doubt trustworthiness.   \n",
       "7   Did not find a reason to doubt trustworthiness.   \n",
       "8   Did not find a reason to doubt trustworthiness.   \n",
       "9   Did not find a reason to doubt trustworthiness.   \n",
       "10  Did not find a reason to doubt trustworthiness.   \n",
       "11  Did not find a reason to doubt trustworthiness.   \n",
       "12  Did not find a reason to doubt trustworthiness.   \n",
       "13                                             None   \n",
       "14  Did not find a reason to doubt trustworthiness.   \n",
       "15                                             None   \n",
       "\n",
       "                                       pii_items_list  \n",
       "0      [John Doe, john.doe@example.com, 555-123-4567]  \n",
       "1                [4111111111111112, 4111111111111111]  \n",
       "2                                       [987-65-4321]  \n",
       "3                      [1234, jane.smith@example.net]  \n",
       "4   [jane.doe@domain.com, 555-234-5678, 555-234-5677]  \n",
       "5               [sk_test_51HfVxECq4sTBwnjJd123T34ef7]  \n",
       "6                          [456-78-9123, 716-72-4618]  \n",
       "7     [4000 0000 0000 0002, mark.stevens@example.com]  \n",
       "8   [555-23-5677, 4111111111111112, Alex Jones, al...  \n",
       "9                    [sk_live_2sdDfjWErLfjweTso392Dd]  \n",
       "10                                                 []  \n",
       "11                                                 []  \n",
       "12                                                 []  \n",
       "13                                      [123-45-5678]  \n",
       "14                                                 []  \n",
       "15                                [+1 (123) 456-7890]  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Show final dataframe\n",
    "df_original[[\"text\", \"pii_items_json\", \"pii_items_score\", \"pii_items_expl\", \"pii_items_list\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "029f5189-f462-4a5d-8e8f-70520d5e18bf",
   "metadata": {},
   "source": [
    "## Redact PII Items from Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "51b015f0-3724-4eac-b950-b7db6844601b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# You can then redact this PII from the text\n",
    "def redact_pii(row):\n",
    "    # Parse pii items LLM response string\n",
    "    pii_json = row['pii_items_json']\n",
    "    # Get all pii items\n",
    "    pii_items = get_all_values(pii_json)\n",
    "    # Iterate over each PII item in the list\n",
    "    text = row['text']\n",
    "    for pii in pii_items:\n",
    "        # Replace each occurrence of the PII item with a string of 'x's of the same length\n",
    "        redacted_pii = 'x' * len(pii)\n",
    "        if pii in text:\n",
    "            text = text.replace(pii, redacted_pii)\n",
    "        else:\n",
    "            print(f\"WARNING: PII Item: {pii} not found in text. Please remove manually.\")\n",
    "    return text\n",
    "\n",
    "# Apply the redact_pii function to each row\n",
    "df_original['redacted_text'] = df_original.apply(redact_pii, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "331c936e-6553-4f17-b11a-055ae850dd01",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Customer: Hi, my name is John Doe. I need help with my account. Can you check it using my email john.doe@example.com and my phone number 555-123-4567?\n",
      "\n",
      "Customer: Hi, my name is xxxxxxxx. I need help with my account. Can you check it using my email xxxxxxxxxxxxxxxxxxxx and my phone number xxxxxxxxxxxx?\n",
      "\n",
      "-------------------------------------------\n",
      "\n",
      "Customer: Hello, I want to update my credit card information. My old card is 4111111111111112 and my current card number is 4111111111111111.\n",
      "\n",
      "Customer: Hello, I want to update my credit card information. My old card is xxxxxxxxxxxxxxxx and my current card number is xxxxxxxxxxxxxxxx.\n",
      "\n",
      "-------------------------------------------\n",
      "\n",
      "Agent: I see you want to reset your account. Can you confirm your SSN? Customer: Sure, it's 987-65-4321.\n",
      "\n",
      "Agent: I see you want to reset your account. Can you confirm your SSN? Customer: Sure, it's xxxxxxxxxxx.\n",
      "\n",
      "-------------------------------------------\n",
      "\n",
      "Customer: I received a charge on my card ending in 1234. Can you check why this happened? My email is jane.smith@example.net.\n",
      "\n",
      "Customer: I received a charge on my card ending in xxxx. Can you check why this happened? My email is xxxxxxxxxxxxxxxxxxxxxx.\n",
      "\n",
      "-------------------------------------------\n",
      "\n",
      "Customer: My phone numbers 555-234-5678 and 555-234-5677 aren’t associated with my account anymore. Can you update it with jane.doe@domain.com?\n",
      "\n",
      "Customer: My phone numbers xxxxxxxxxxxx and xxxxxxxxxxxx aren’t associated with my account anymore. Can you update it with xxxxxxxxxxxxxxxxxxx?\n",
      "\n",
      "-------------------------------------------\n",
      "\n",
      "Agent: For security purposes, we cannot disclose API keys over chat. Customer: Here is mine: sk_test_51HfVxECq4sTBwnjJd123T34ef7.\n",
      "\n",
      "Agent: For security purposes, we cannot disclose API keys over chat. Customer: Here is mine: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.\n",
      "\n",
      "-------------------------------------------\n",
      "\n",
      "Customer: I’m concerned because my husband and I's ssns 456-78-9123 and 716-72-4618 were submitted on your site, but I didn't get confirmation.\n",
      "\n",
      "Customer: I’m concerned because my husband and I's ssns xxxxxxxxxxx and xxxxxxxxxxx were submitted on your site, but I didn't get confirmation.\n",
      "\n",
      "-------------------------------------------\n",
      "\n",
      "Customer: I think someone stole my information. My email is mark.stevens@example.com, and I got suspicious activity on card 4000 0000 0000 0002.\n",
      "\n",
      "Customer: I think someone stole my information. My email is xxxxxxxxxxxxxxxxxxxxxxxx, and I got suspicious activity on card xxxxxxxxxxxxxxxxxxx.\n",
      "\n",
      "-------------------------------------------\n",
      "\n",
      "Customer: Here is all my info: 555-23-5677, 4111111111111112, Alex Jones, alex.jones@example.org, 555-789-1234, cl_2sdDfjWErLfjweTso392Dd\n",
      "\n",
      "Customer: Here is all my info: xxxxxxxxxxx, xxxxxxxxxxxxxxxx, xxxxxxxxxx, xxxxxxxxxxxxxxxxxxxxxx, xxxxxxxxxxxx, xxxxxxxxxxxxxxxxxxxxxxxxx\n",
      "\n",
      "-------------------------------------------\n",
      "\n",
      "Customer: My API key for service is sk_live_2sdDfjWErLfjweTso392Dd, and I am facing issues while using it.\n",
      "\n",
      "Customer: My API key for service is xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx, and I am facing issues while using it.\n",
      "\n",
      "-------------------------------------------\n",
      "\n",
      "Customer: Hi! I’m having some issues accessing my account. Could you assist me? Agent: Hello! Sure, I'd be happy to help. Could you describe the issue you're experiencing? Customer: Every time I try to log in, I get an error message saying, “Unable to authenticate.” I’m sure I’m entering the right password. Agent: Got it. That error typically appears if there’s a password issue or if multi-factor authentication needs updating. Have you tried resetting your password? Customer: Not yet. If I reset it, will I lose any saved data? Agent: Not at all! Resetting your password only changes your login credentials; all your saved data will remain intact. Customer: Perfect! Can you guide me through the password reset process? Agent: Absolutely. First, head to our login page and click on “Forgot Password.” Then, follow the prompts to enter the email associated with your account. You’ll receive a reset link in your email.\n",
      "\n",
      "Customer: Hi! I’m having some issues accessing my account. Could you assist me? Agent: Hello! Sure, I'd be happy to help. Could you describe the issue you're experiencing? Customer: Every time I try to log in, I get an error message saying, “Unable to authenticate.” I’m sure I’m entering the right password. Agent: Got it. That error typically appears if there’s a password issue or if multi-factor authentication needs updating. Have you tried resetting your password? Customer: Not yet. If I reset it, will I lose any saved data? Agent: Not at all! Resetting your password only changes your login credentials; all your saved data will remain intact. Customer: Perfect! Can you guide me through the password reset process? Agent: Absolutely. First, head to our login page and click on “Forgot Password.” Then, follow the prompts to enter the email associated with your account. You’ll receive a reset link in your email.\n",
      "\n",
      "-------------------------------------------\n",
      "\n",
      "Customer: Hi, I need help with an order issue. I ordered something on 09/15/2023, and I still haven’t received it. My order number is 2048573. Agent: I’m here to help! Let me check the status of order #2048573 for you. One moment please. Customer: Sure, thanks! Agent: Thank you for waiting. It looks like your order was shipped on 09/18/2023 and is currently in transit. You should receive it by 09/25/2023. Customer: Got it! Will I be able to track it? Agent: Yes, you can! Your tracking number is 1Z999AA10123456784. Just go to our tracking page, enter the number, and you’ll see the latest status. Customer: Great, thank you. One more thing—I saw a charge of $19.99 on my account. Could you explain what it’s for? Agent: Sure thing. That charge on 09/20/2023 was for a subscription renewal. Let me know if you'd like to update your subscription settings. Customer: I’d like to change it to the monthly plan instead of the annual one. Agent: No problem! I’ll switch your subscription to the monthly plan, which is $9.99 per month, starting with your next billing cycle.\n",
      "\n",
      "Customer: Hi, I need help with an order issue. I ordered something on 09/15/2023, and I still haven’t received it. My order number is 2048573. Agent: I’m here to help! Let me check the status of order #2048573 for you. One moment please. Customer: Sure, thanks! Agent: Thank you for waiting. It looks like your order was shipped on 09/18/2023 and is currently in transit. You should receive it by 09/25/2023. Customer: Got it! Will I be able to track it? Agent: Yes, you can! Your tracking number is 1Z999AA10123456784. Just go to our tracking page, enter the number, and you’ll see the latest status. Customer: Great, thank you. One more thing—I saw a charge of $19.99 on my account. Could you explain what it’s for? Agent: Sure thing. That charge on 09/20/2023 was for a subscription renewal. Let me know if you'd like to update your subscription settings. Customer: I’d like to change it to the monthly plan instead of the annual one. Agent: No problem! I’ll switch your subscription to the monthly plan, which is $9.99 per month, starting with your next billing cycle.\n",
      "\n",
      "-------------------------------------------\n",
      "\n",
      "Customer: Hi, I have a question about canceling my service. Agent: Hello! I can assist you with that. Just to confirm, are you looking to cancel immediately or at the end of the current billing period? Customer: At the end of the period would be good. Agent: Understood! Before I proceed, I need to read a detailed disclaimer regarding our cancellation policy. Here it is: 'By proceeding with the cancellation of your service, you agree to and acknowledge the following terms: (1) All account data, including but not limited to settings, preferences, saved information, and any content generated through our platform, will be permanently deleted 30 days after the end of your current billing period. It is your responsibility to back up any important data before this date. (2) All pending charges for services rendered up to the cancellation date will remain due and payable, and failure to settle these charges may result in additional fees. (3) Any active warranties or service guarantees that were applicable under your account will become null and void upon the effective cancellation date. (4) By canceling, you forfeit access to all promotional benefits, loyalty rewards, or other incentives previously applied to your account, and any accumulated points or credits will also be forfeited. (5) Our team will no longer be able to provide support for retrieving data, reactivating prior settings, or addressing issues related to the service post-cancellation. (6) Re-enrollment following cancellation will be treated as a new account registration, and any prior status, discounts, or service history may not be reinstated. Please review the full policy on our website for complete details. By choosing to continue, you affirm that you have read, understood, and agree to these terms in their entirety.' Customer: That’s really thorough! I understand the terms. Let’s go ahead with the cancellation. Agent: Thank you for confirming. I’ll schedule your cancellation for the end of your current billing period. Please let us know if there’s anything else we can assist with before then!\n",
      "\n",
      "Customer: Hi, I have a question about canceling my service. Agent: Hello! I can assist you with that. Just to confirm, are you looking to cancel immediately or at the end of the current billing period? Customer: At the end of the period would be good. Agent: Understood! Before I proceed, I need to read a detailed disclaimer regarding our cancellation policy. Here it is: 'By proceeding with the cancellation of your service, you agree to and acknowledge the following terms: (1) All account data, including but not limited to settings, preferences, saved information, and any content generated through our platform, will be permanently deleted 30 days after the end of your current billing period. It is your responsibility to back up any important data before this date. (2) All pending charges for services rendered up to the cancellation date will remain due and payable, and failure to settle these charges may result in additional fees. (3) Any active warranties or service guarantees that were applicable under your account will become null and void upon the effective cancellation date. (4) By canceling, you forfeit access to all promotional benefits, loyalty rewards, or other incentives previously applied to your account, and any accumulated points or credits will also be forfeited. (5) Our team will no longer be able to provide support for retrieving data, reactivating prior settings, or addressing issues related to the service post-cancellation. (6) Re-enrollment following cancellation will be treated as a new account registration, and any prior status, discounts, or service history may not be reinstated. Please review the full policy on our website for complete details. By choosing to continue, you affirm that you have read, understood, and agree to these terms in their entirety.' Customer: That’s really thorough! I understand the terms. Let’s go ahead with the cancellation. Agent: Thank you for confirming. I’ll schedule your cancellation for the end of your current billing period. Please let us know if there’s anything else we can assist with before then!\n",
      "\n",
      "-------------------------------------------\n",
      "\n",
      "Agent: I see you want to deactivate your account. Can you confirm your number? Customer: Sure, it's 123-45-5678\n",
      "\n",
      "Agent: I see you want to deactivate your account. Can you confirm your number? Customer: Sure, it's xxxxxxxxxxx\n",
      "\n",
      "-------------------------------------------\n",
      "\n",
      "Agent: Hello! What is your issue? Customer: I need help with my account.\n",
      "\n",
      "Agent: Hello! What is your issue? Customer: I need help with my account.\n",
      "\n",
      "-------------------------------------------\n",
      "\n",
      "Agent: I see you want to check the status of account. Can you confirm your phone number? Customer: Sure, it's +1 (123) 456-7890\n",
      "\n",
      "Agent: I see you want to check the status of account. Can you confirm your phone number? Customer: Sure, it's xxxxxxxxxxxxxxxxx\n",
      "\n",
      "-------------------------------------------\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# View all results\n",
    "for idx, row in df_original.iterrows():\n",
    "    print(row.text)\n",
    "    print()\n",
    "    print(row.redacted_text)\n",
    "    print()\n",
    "    print(\"-------------------------------------------\")\n",
    "    print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}