updating

.gitignore

@@ -1,3 +1,4 @@
 
 .ipynb_checkpoints/AttentionsVisualization-checkpoint.ipynb
 /sample_Data/original_images/.ipynb_checkpoints
+/.ipynb_checkpoints

AttentionsVisualization.ipynb

@@ -161,9 +161,7 @@
     "                    \"an H&E image of smooth muscle\",\n",
     "                    \"an H&E image of normal colon mucosa\",\n",
     "                    \"an H&E image of cancer-associated stroma\",\n",
-    "                    \"an H&E image of colorectal adenocarcinoma epithelium\",\n",
-    "                   \"\"\n",
-    "                   ]"
+    "                    \"an H&E image of colorectal adenocarcinoma epithelium\"]"
    ]
   },
   {

PGD_attack.ipynb

@@ -0,0 +1,203 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "93a1cbcf-3958-4579-bf4c-068b98c2edf7",
+   "metadata": {},
+   "source": [
+    "## Imorts"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "84fbe9f0-0b00-4e66-b730-f7f57b6013e2",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/usr/local/Caskroom/miniconda/base/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "from PIL import Image\n",
+    "import torch\n",
+    "import torch.nn as nn\n",
+    "import torchvision.transforms as transforms\n",
+    "from torchvision.utils import save_image\n",
+    "from transformers import CLIPProcessor, CLIPModel\n",
+    "import cv2\n",
+    "\n",
+    "from skimage import io\n",
+    "from skimage.metrics import structural_similarity as ssim\n",
+    "import numpy as np\n",
+    "import matplotlib.pyplot as plt\n",
+    "from skimage import color\n",
+    "import os\n",
+    "import requests\n",
+    "import torch.nn.functional as nnf\n",
+    "import pandas as pd\n",
+    "from matplotlib.pyplot import imshow"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "id": "f6f1e232-d992-47b5-bd4e-d72439ab2960",
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/usr/local/Caskroom/miniconda/base/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
+      "  return self.fget.__get__(instance, owner)()\n"
+     ]
+    }
+   ],
+   "source": [
+    "model = CLIPModel.from_pretrained(\"vinid/plip\")\n",
+    "processor = CLIPProcessor.from_pretrained(\"vinid/plip\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "88eb8f11-14f0-4bbe-8b11-6f23aea85da6",
+   "metadata": {},
+   "source": [
+    "## Attack Method"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "id": "0f24ea5a-382b-4f6c-ba29-ec2cab225746",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def pgd_attack(fileName):\n",
+    "    # Load and process the image\n",
+    "    image_path = fileName\n",
+    "    image_file = fileName.split('/')[-1]\n",
+    "    image_name = image_file.split('.')\n",
+    "    image = Image.open(image_path).convert(\"RGB\")\n",
+    "    image = image.resize((224, 224))\n",
+    "\n",
+    "    # Text prompts\n",
+    "    text_prompts = [\"an H&E image of adipose\",\"an H&E image of background\",\"an H&E image of debris\",\"an H&E image of lymphocytes\",\"an H&E image of mucus\", \"an H&E image of smooth muscle\",\"an H&E image of normal colon mucosa\",\"an H&E image of cancer-associated stroma\",\"an H&E image of colorectal adenocarcinoma epithelium\"]\n",
+    "\n",
+    "    # Process image and text\n",
+    "    inputs = processor(text=text_prompts, images=image, return_tensors=\"pt\", padding=True)\n",
+    "    image_tensor = inputs['pixel_values']\n",
+    "    image_tensor = (image_tensor - image_tensor.min()) / (image_tensor.max() - image_tensor.min())\n",
+    "    inputs['pixel_values'] = image_tensor\n",
+    "    inputs.pixel_values.requires_grad_(True)\n",
+    "\n",
+    "    # Get the model outputs for original image\n",
+    "    outputs_orig = model(**inputs)\n",
+    "    logits_per_image = outputs_orig.logits_per_image  # this is the image-text similarity score\n",
+    "    original_probs = nnf.softmax(logits_per_image, dim=1)\n",
+    "    top_p, top_class = original_probs.topk(1, dim = 1) \n",
+    "\n",
+    "    for p in model.parameters():\n",
+    "        p.requires_grad = False\n",
+    "\n",
+    "    image = inputs[\"pixel_values\"]\n",
+    "    image_original_np = image.detach().cpu().numpy()\n",
+    "    true_label = torch.argmax(original_probs)\n",
+    "    target_label = 8\n",
+    "\n",
+    "    LR = 0.3 #LR for the image changes\n",
+    "    steps = 10 #the number of steps to get adversary\n",
+    "\n",
+    "    for step in range(steps):\n",
+    "        image_np = inputs[\"pixel_values\"].detach().cpu().numpy()\n",
+    "        criterion = torch.nn.CrossEntropyLoss()\n",
+    "        labels = torch.Tensor([target_label]).reshape([1]).to(\"cpu\")\n",
+    "        inputs[\"pixel_values\"].requires_grad = True\n",
+    "        outputs_adv = model(**inputs)\n",
+    "        logits_per_image_adv = outputs_adv.logits_per_image\n",
+    "        adv_probs = nnf.softmax(logits_per_image_adv, dim=1)\n",
+    "        top_p_adv, top_class_adv = adv_probs.topk(1, dim = 1)\n",
+    "        loss = criterion(outputs_adv.logits_per_image, labels.long())\n",
+    "        inputs[\"pixel_values\"].retain_grad()\n",
+    "        loss.retain_grad()\n",
+    "        loss.backward()\n",
+    "        loss_out_np = loss.data.detach().cpu().numpy()\n",
+    "\n",
+    "        if step == steps - 1:  # Check if it's the last step\n",
+    "            final_adversarial_np = image_np[0].transpose([1,2,0])\n",
+    "            final_adversarial_np = np.clip(final_adversarial_np * 255, 0, 255).astype(np.uint8)\n",
+    "            final_adversarial_image = Image.fromarray(final_adversarial_np)\n",
+    "            \n",
+    "            final_noise = image_np[0]-image_original_np[0]\n",
+    "            pert = np.clip(final_noise * 255, 0, 255).astype(np.uint8)\n",
+    "            pert_image = Image.fromarray(pert.transpose([1,2,0]))\n",
+    "\n",
+    "            \n",
+    "            # imshow(np.asarray(pert_image))\n",
+    "            # imshow(np.asarray(final_adversarial_image))\n",
+    "            plt.imshow(final_adversarial_image)\n",
+    "            \n",
+    "        image_grad = inputs[\"pixel_values\"].grad.detach().cpu().numpy()\n",
+    "        image_np = inputs['pixel_values'].detach().cpu().numpy() - LR*image_grad #taking the image step\n",
+    "        inputs['pixel_values'] = torch.Tensor(image_np)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "63adb1a7-dff8-489e-adcb-e69fa9f6cfaf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "directory_path = './sample_Data/original_images'\n",
+    "\n",
+    "\n",
+    "image_files = os.listdir(directory_path)\n",
+    "image_files = sorted(image_files)\n",
+    "\n",
+    "for i, image_file in enumerate(image_files):\n",
+    "    if(image_file.endswith('.tif')):\n",
+    "        img_path = os.path.join(directory_path, image_file)\n",
+    "        pgd_attack(img_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a55b3727-12fb-4072-9b0a-83f8d75069dd",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}