Blip_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "GitHub  \n",
    "https://github.com/salesforce/BLIP  \n",
    "論文  \n",
    "https://arxiv.org/abs/2201.12086v1  \n",
    "  \n",
    "<a href=\"https://colab.research.google.com/github/kaz12tech/ai_demos/blob/master/Blip_demo.ipynb\" target=\"_blank\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ライブラリのインストール・GitHubからCode Clone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%cd /content/\n",
    "\n",
    "import sys\n",
    "if 'google.colab' in sys.modules:\n",
    "    print('Running in Colab.')\n",
    "    !pip3 install transformers==4.15.0 timm==0.4.12 fairscale==0.4.4\n",
    "    !git clone https://github.com/salesforce/BLIP\n",
    "    %cd BLIP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# テスト画像のロード\n",
    "upload選択時、アップロードした画像を使用\n",
    "sample選択時、salesforce提供のサンプル画像をwebからロードして使用"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import requests\n",
    "import torch\n",
    "from torchvision import transforms\n",
    "from torchvision.transforms.functional import InterpolationMode\n",
    "from google.colab import files\n",
    "\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "\n",
    "#@markdown sample or uploadを選択\n",
    "image_type ='upload' #@param ['sample', 'upload']\n",
    "if image_type == 'sample':\n",
    "    img_url = 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/demo.jpg' \n",
    "    raw_image = Image.open(requests.get(img_url, stream=True).raw).convert('RGB')\n",
    "else:\n",
    "    uploaded = files.upload()\n",
    "    uploaded = list(uploaded.keys())\n",
    "    file_name = uploaded[0]\n",
    "    raw_image = Image.open(file_name).convert('RGB')\n",
    "\n",
    "\n",
    "w,h = raw_image.size\n",
    "display(raw_image.resize((w//5,h//5)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Image Captioningモデルのロード"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from models.blip import blip_decoder\n",
    "\n",
    "image_size = 384\n",
    "transform = transforms.Compose([\n",
    "    transforms.Resize((image_size,image_size),interpolation=InterpolationMode.BICUBIC),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711))\n",
    "    ]) \n",
    "image = transform(raw_image).unsqueeze(0).to(device)     \n",
    "\n",
    "model_url = 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model*_base_caption.pth'\n",
    "    \n",
    "model = blip_decoder(pretrained=model_url, image_size=384, vit='base')\n",
    "model.eval()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Image Captioning\n",
    "finetuneしたBLIPモデルを使用して画像のキャプションを予測"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with torch.no_grad():\n",
    "    caption = model.generate(image, sample=False, num_beams=3, max_length=20, min_length=5)\n",
    "    print('caption: '+caption[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# visual question answering(VQA)モデルのロード"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from models.blip_vqa import blip_vqa\n",
    "\n",
    "image_size = 480\n",
    "transform = transforms.Compose([\n",
    "    transforms.Resize((image_size,image_size),interpolation=InterpolationMode.BICUBIC),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711))\n",
    "    ]) \n",
    "image = transform(raw_image).unsqueeze(0).to(device)        \n",
    "\n",
    "model_url = 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model*_vqa.pth'\n",
    "    \n",
    "model = blip_vqa(pretrained=model_url, image_size=480, vit='base')\n",
    "model.eval()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#VQA\n",
    "finetuneしたBLIPモデルを使用してVQAを予測"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#@title Question設定\n",
    "#@markdown 画像に対する質問を英語で記載してください。\n",
    "question = 'where is the woman sitting?' #@param {type:\"string\"}\n",
    "\n",
    "with torch.no_grad():\n",
    "    answer = model(image, question, train=False, inference='generate') \n",
    "    print('answer: '+answer[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# feature extraction(特徴抽出)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from models.blip import blip_feature_extractor\n",
    "\n",
    "image_size = 224\n",
    "transform = transforms.Compose([\n",
    "    transforms.Resize((image_size,image_size),interpolation=InterpolationMode.BICUBIC),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711))\n",
    "    ]) \n",
    "image = transform(raw_image).unsqueeze(0).to(device)     \n",
    "\n",
    "model_url = 'https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base.pth'\n",
    "    \n",
    "model = blip_feature_extractor(pretrained=model_url, image_size=224, vit='base')\n",
    "model.eval()\n",
    "model = model.to(device)\n",
    "\n",
    "caption = 'a woman sitting on the beach with a dog'\n",
    "\n",
    "multimodal_feature = model(image, caption, mode='multimodal')[0,0]\n",
    "image_feature = model(image, caption, mode='image')[0,0]\n",
    "text_feature = model(image, caption, mode='text')[0,0]"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}