OFA_demo.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DHx7Q9pgsArb"
      },
      "source": [
        "論文<br>\n",
        "https://arxiv.org/abs/2202.03052<br>\n",
        "<br>\n",
        "GitHub<br>\n",
        "https://github.com/OFA-Sys/OFA<br>\n",
        "<br>\n",
        "<a href=\"https://colab.research.google.com/github/kaz12tech/ai_demos/blob/master/OFA_demo.ipynb\" target=\"_blank\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p5ZoRBinsArg"
      },
      "source": [
        "## 環境セットアップ"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0I-tpdoysArg"
      },
      "source": [
        "## GPU確認"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2PcII5fLdHnJ",
        "outputId": "58db7866-1440-4eed-ae10-1f0349100a9b"
      },
      "outputs": [],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7aSrsNFjsAri"
      },
      "source": [
        "## GitHubからコード取得"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R-Y5auuvnnnT",
        "outputId": "ec12a2c9-00bc-47db-e300-e969f6bbab82"
      },
      "outputs": [],
      "source": [
        "%cd /content\n",
        "\n",
        "!git clone https://github.com/OFA-Sys/OFA.git"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rUnFqC99sArk"
      },
      "source": [
        "## 学習済みモデルのダウンロード"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WYJJ2yVyszKW",
        "outputId": "a36a256b-84c1-4afb-b155-700b854f7ca4"
      },
      "outputs": [],
      "source": [
        "%cd /content\n",
        "\n",
        "!mkdir -p /content/OFA/checkpoints/\n",
        "!wget https://ofa-silicon.oss-us-west-1.aliyuncs.com/checkpoints/ofa_large_clean.pt\n",
        "!mv ofa_large_clean.pt OFA/checkpoints/ofa_large.pt"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zivQ5SrJsArj"
      },
      "source": [
        "## ライブラリのインストール"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NzRfmkF5tEGN",
        "outputId": "11118544-6e56-41e6-bb30-9acd1e4f397e"
      },
      "outputs": [],
      "source": [
        "# fairseq\n",
        "# RESTART RUNTIMEが表示された場合「ランタイムを再起動」\n",
        "%cd /content\n",
        "!git clone https://github.com/pytorch/fairseq.git\n",
        "\n",
        "%cd /content/fairseq\n",
        "!pip install --use-feature=in-tree-build ./"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yK4oROh_uOfl",
        "outputId": "aebdc5e7-453f-4fd8-b0a4-d3fb90d758bb"
      },
      "outputs": [],
      "source": [
        "%cd /content/OFA\n",
        "\n",
        "# 1行目は削除\n",
        "!sed '1d' requirements.txt | xargs -I {} pip install {}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wqixwQG5sArj"
      },
      "source": [
        "## ライブラリのインポート"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yIr59B2uu3aZ"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import numpy as np\n",
        "from fairseq import checkpoint_utils, distributed_utils, options, tasks, utils\n",
        "from fairseq.dataclass.utils import convert_namespace_to_omegaconf\n",
        "from tasks.mm_tasks.refcoco import RefcocoTask\n",
        "\n",
        "from models.ofa import OFAModel\n",
        "from PIL import Image\n",
        "\n",
        "import cv2\n",
        "import numpy\n",
        "from google.colab.patches import cv2_imshow\n",
        "\n",
        "tasks.register_task('refcoco', RefcocoTask)\n",
        "\n",
        "# turn on cuda if GPU is available\n",
        "use_cuda = torch.cuda.is_available()\n",
        "# use fp16 only when GPU is available\n",
        "use_fp16 = False\n",
        "\n",
        "# specify some options for evaluation\n",
        "parser = options.get_generation_parser()\n",
        "input_args = [\"\", \"--task=refcoco\", \"--beam=10\", \"--path=checkpoints/ofa_large.pt\", \"--bpe-dir=utils/BPE\"]\n",
        "args = options.parse_args_and_arch(parser, input_args)\n",
        "cfg = convert_namespace_to_omegaconf(args)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J7lUWV4yvJ5K"
      },
      "source": [
        "# モデルビルド"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iJrKsuCLvR95",
        "outputId": "65123c7a-94c9-4158-83e9-6bc0cb2345b7"
      },
      "outputs": [],
      "source": [
        "# configファイルと学習済みモデルのロード\n",
        "task = tasks.setup_task(cfg.task)\n",
        "models, cfg = checkpoint_utils.load_model_ensemble(\n",
        "    utils.split_paths(cfg.common_eval.path),\n",
        "    task=task\n",
        ")\n",
        "\n",
        "# GPUに載せる\n",
        "for model in models:\n",
        "    model.eval()\n",
        "    if use_fp16:\n",
        "        model.half()\n",
        "    if use_cuda and not cfg.distributed_training.pipeline_model_parallel:\n",
        "        model.cuda()\n",
        "    model.prepare_for_inference_(cfg)\n",
        "\n",
        "# generatorの初期化\n",
        "generator = task.build_generator(models, cfg.generation)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IKuVCxY1s0ah"
      },
      "source": [
        "# Preprocess\n",
        "transformation定義"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G2EA83knv6z4",
        "outputId": "0da4ca85-40f2-4305-8346-be952ef06dd5"
      },
      "outputs": [],
      "source": [
        "# Image transform\n",
        "from torchvision import transforms\n",
        "mean = [0.5, 0.5, 0.5]\n",
        "std = [0.5, 0.5, 0.5]\n",
        "\n",
        "patch_resize_transform = transforms.Compose([\n",
        "    lambda image: image.convert(\"RGB\"),\n",
        "    transforms.Resize((task.cfg.patch_image_size, task.cfg.patch_image_size), interpolation=Image.BICUBIC),\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize(mean=mean, std=std),\n",
        "])\n",
        "\n",
        "# Text preprocess\n",
        "bos_item = torch.LongTensor([task.src_dict.bos()])\n",
        "eos_item = torch.LongTensor([task.src_dict.eos()])\n",
        "pad_idx = task.src_dict.pad()\n",
        "\n",
        "\n",
        "def get_symbols_to_strip_from_output(generator):\n",
        "    if hasattr(generator, \"symbols_to_strip_from_output\"):\n",
        "        return generator.symbols_to_strip_from_output\n",
        "    else:\n",
        "        return {generator.bos, generator.eos}\n",
        "\n",
        "\n",
        "def decode_fn(x, tgt_dict, bpe, generator, tokenizer=None):\n",
        "    x = tgt_dict.string(x.int().cpu(), extra_symbols_to_ignore=get_symbols_to_strip_from_output(generator))\n",
        "    token_result = []\n",
        "    bin_result = []\n",
        "    img_result = []\n",
        "    for token in x.strip().split():\n",
        "      if token.startswith('<bin_'):\n",
        "        bin_result.append(token)\n",
        "      elif token.startswith('<code_'):\n",
        "        img_result.append(token)\n",
        "      else:\n",
        "        if bpe is not None:\n",
        "          token = bpe.decode('{}'.format(token))\n",
        "        if tokenizer is not None:\n",
        "          token = tokenizer.decode(token)\n",
        "        if token.startswith(' ') or len(token_result) == 0:\n",
        "          token_result.append(token.strip())\n",
        "        else:\n",
        "          token_result[-1] += token\n",
        "\n",
        "    return ' '.join(token_result), ' '.join(bin_result), ' '.join(img_result)\n",
        "\n",
        "\n",
        "def coord2bin(coords, w_resize_ratio, h_resize_ratio):\n",
        "    coord_list = [float(coord) for coord in coords.strip().split()]\n",
        "    bin_list = []\n",
        "    bin_list += [\"<bin_{}>\".format(int((coord_list[0] * w_resize_ratio / task.cfg.max_image_size * (task.cfg.num_bins - 1))))]\n",
        "    bin_list += [\"<bin_{}>\".format(int((coord_list[1] * h_resize_ratio / task.cfg.max_image_size * (task.cfg.num_bins - 1))))]\n",
        "    bin_list += [\"<bin_{}>\".format(int((coord_list[2] * w_resize_ratio / task.cfg.max_image_size * (task.cfg.num_bins - 1))))]\n",
        "    bin_list += [\"<bin_{}>\".format(int((coord_list[3] * h_resize_ratio / task.cfg.max_image_size * (task.cfg.num_bins - 1))))]\n",
        "    return ' '.join(bin_list)\n",
        "\n",
        "\n",
        "def bin2coord(bins, w_resize_ratio, h_resize_ratio):\n",
        "    bin_list = [int(bin[5:-1]) for bin in bins.strip().split()]\n",
        "    coord_list = []\n",
        "    coord_list += [bin_list[0] / (task.cfg.num_bins - 1) * task.cfg.max_image_size / w_resize_ratio]\n",
        "    coord_list += [bin_list[1] / (task.cfg.num_bins - 1) * task.cfg.max_image_size / h_resize_ratio]\n",
        "    coord_list += [bin_list[2] / (task.cfg.num_bins - 1) * task.cfg.max_image_size / w_resize_ratio]\n",
        "    coord_list += [bin_list[3] / (task.cfg.num_bins - 1) * task.cfg.max_image_size / h_resize_ratio]\n",
        "    return coord_list\n",
        "\n",
        "\n",
        "def encode_text(text, length=None, append_bos=False, append_eos=False):\n",
        "    line = [\n",
        "      task.bpe.encode(' {}'.format(word.strip())) \n",
        "      if not word.startswith('<code_') and not word.startswith('<bin_') else word\n",
        "      for word in text.strip().split()\n",
        "    ]\n",
        "    line = ' '.join(line)\n",
        "    s = task.tgt_dict.encode_line(\n",
        "        line=line,\n",
        "        add_if_not_exist=False,\n",
        "        append_eos=False\n",
        "    ).long()\n",
        "    if length is not None:\n",
        "        s = s[:length]\n",
        "    if append_bos:\n",
        "        s = torch.cat([bos_item, s])\n",
        "    if append_eos:\n",
        "        s = torch.cat([s, eos_item])\n",
        "    return s\n",
        "\n",
        "def construct_sample(image: Image, instruction: str):\n",
        "    patch_image = patch_resize_transform(image).unsqueeze(0)\n",
        "    patch_mask = torch.tensor([True])\n",
        "\n",
        "    instruction = encode_text(' {}'.format(instruction.lower().strip()), append_bos=True, append_eos=True).unsqueeze(0)\n",
        "    instruction_length = torch.LongTensor([s.ne(pad_idx).long().sum() for s in instruction])\n",
        "    sample = {\n",
        "        \"id\":np.array(['42']),\n",
        "        \"net_input\": {\n",
        "            \"src_tokens\": instruction,\n",
        "            \"src_lengths\": instruction_length,\n",
        "            \"patch_images\": patch_image,\n",
        "            \"patch_masks\": patch_mask,\n",
        "        }\n",
        "    }\n",
        "    return sample\n",
        "  \n",
        "# Function to turn FP32 to FP16\n",
        "def apply_half(t):\n",
        "    if t.dtype is torch.float32:\n",
        "        return t.to(dtype=torch.half)\n",
        "    return t"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DJCDpxSfxwWd"
      },
      "source": [
        "# Image Captioning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gpUf7biLxz7b"
      },
      "source": [
        "## テスト画像取得"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "u2OnnMC-xuOb",
        "outputId": "372f8e86-dc37-4766-f6af-658e4740e58b"
      },
      "outputs": [],
      "source": [
        "%cd /content/OFA\n",
        "!mkdir test_imgs\n",
        "%cd test_imgs\n",
        "\n",
        "# https://www.pakutaso.com/20180239038post-15116.html\n",
        "!wget https://www.pakutaso.com/shared/img/thumb/smIMGL4174_TP_V4.jpg\n",
        "\n",
        "%cd /content/OFA\n",
        "image = Image.open('/content/OFA/test_imgs/smIMGL4174_TP_V4.jpg')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 747
        },
        "id": "WbUbu7Ncy-Ew",
        "outputId": "e656ea4e-df07-446b-e7f2-d1ef65d99bbf"
      },
      "outputs": [],
      "source": [
        "instruction = \"what does the image describe?\"\n",
        "\n",
        "# Construct input sample & preprocess for GPU if cuda available\n",
        "sample = construct_sample(image, instruction)\n",
        "sample = utils.move_to_cuda(sample) if use_cuda else sample\n",
        "sample = utils.apply_to_sample(apply_half, sample) if use_fp16 else sample\n",
        "\n",
        "# Generate result\n",
        "with torch.no_grad():\n",
        "    hypos = task.inference_step(generator, models, sample)\n",
        "    tokens1, bins1, imgs1 = decode_fn(hypos[0][0][\"tokens\"], task.tgt_dict, task.bpe, generator)\n",
        "    tokens2, bins2, imgs2 = decode_fn(hypos[0][1][\"tokens\"], task.tgt_dict, task.bpe, generator)\n",
        "    tokens3, bins3, imgs3 = decode_fn(hypos[0][2][\"tokens\"], task.tgt_dict, task.bpe, generator)\n",
        "    tokens4, bins4, imgs4 = decode_fn(hypos[0][3][\"tokens\"], task.tgt_dict, task.bpe, generator)\n",
        "    tokens5, bins5, imgs5 = decode_fn(hypos[0][4][\"tokens\"], task.tgt_dict, task.bpe, generator)\n",
        "\n",
        "# display result\n",
        "display(image)\n",
        "print('Instruction: {}'.format(instruction))\n",
        "print('OFA\\'s Output1: {}, Probs: {}'.format(tokens1, hypos[0][0][\"score\"].exp().item()))\n",
        "print('OFA\\'s Output2: {}, Probs: {}'.format(tokens2, hypos[0][1][\"score\"].exp().item()))\n",
        "print('OFA\\'s Output3: {}, Probs: {}'.format(tokens3, hypos[0][2][\"score\"].exp().item()))\n",
        "print('OFA\\'s Output4: {}, Probs: {}'.format(tokens4, hypos[0][3][\"score\"].exp().item()))\n",
        "print('OFA\\'s Output5: {}, Probs: {}'.format(tokens5, hypos[0][4][\"score\"].exp().item()))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vPIYd1wY0lK4"
      },
      "source": [
        "# Visual Question Answering: VQA"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wkyf00Vo01CQ"
      },
      "source": [
        "## テスト画像取得"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TcCeor-Q03aT",
        "outputId": "298184c5-3dff-440e-80b2-5bd5135d5f2f"
      },
      "outputs": [],
      "source": [
        "%cd /content/OFA\n",
        "!mkdir test_imgs\n",
        "%cd test_imgs\n",
        "\n",
        "# https://www.pakutaso.com/20180239038post-15116.html\n",
        "!wget https://www.pakutaso.com/shared/img/thumb/smIMGL4174_TP_V4.jpg\n",
        "\n",
        "%cd /content/OFA\n",
        "image = Image.open('/content/OFA/test_imgs/smIMGL4174_TP_V4.jpg')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OvEaEstL07EW"
      },
      "source": [
        "## Question設定"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kSk-FqKT0-Up"
      },
      "outputs": [],
      "source": [
        "instruction = \"What does a woman have in her left hand?\" #@param {type:\"string\"}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 747
        },
        "id": "UudIip4W1T9S",
        "outputId": "c60aa95e-3387-4e6c-da6e-7c661fc6bf79"
      },
      "outputs": [],
      "source": [
        "# Construct input sample & preprocess for GPU if cuda available\n",
        "sample = construct_sample(image, instruction)\n",
        "sample = utils.move_to_cuda(sample) if use_cuda else sample\n",
        "sample = utils.apply_to_sample(apply_half, sample) if use_fp16 else sample\n",
        "\n",
        "# Generate result\n",
        "with torch.no_grad():\n",
        "    hypos = task.inference_step(generator, models, sample)\n",
        "    tokens1, bins1, imgs1 = decode_fn(hypos[0][0][\"tokens\"], task.tgt_dict, task.bpe, generator)\n",
        "    tokens2, bins2, imgs2 = decode_fn(hypos[0][1][\"tokens\"], task.tgt_dict, task.bpe, generator)\n",
        "    tokens3, bins3, imgs3 = decode_fn(hypos[0][2][\"tokens\"], task.tgt_dict, task.bpe, generator)\n",
        "    tokens4, bins4, imgs4 = decode_fn(hypos[0][3][\"tokens\"], task.tgt_dict, task.bpe, generator)\n",
        "    tokens5, bins5, imgs5 = decode_fn(hypos[0][4][\"tokens\"], task.tgt_dict, task.bpe, generator)\n",
        "\n",
        "# display result\n",
        "display(image)\n",
        "print('Instruction: {}'.format(instruction))\n",
        "print('OFA\\'s Output1: {}, Probs: {}'.format(tokens1, hypos[0][0][\"score\"].exp().item()))\n",
        "print('OFA\\'s Output2: {}, Probs: {}'.format(tokens2, hypos[0][1][\"score\"].exp().item()))\n",
        "print('OFA\\'s Output3: {}, Probs: {}'.format(tokens3, hypos[0][2][\"score\"].exp().item()))\n",
        "print('OFA\\'s Output4: {}, Probs: {}'.format(tokens4, hypos[0][3][\"score\"].exp().item()))\n",
        "print('OFA\\'s Output5: {}, Probs: {}'.format(tokens5, hypos[0][4][\"score\"].exp().item()))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vsaj_p6g2p-X"
      },
      "source": [
        "# Grounded QA"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2JMZVhD-23O9"
      },
      "source": [
        "## テスト画像取得"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "m5qCAIIM244y"
      },
      "outputs": [],
      "source": [
        "%cd /content/OFA\n",
        "!mkdir test_imgs\n",
        "%cd test_imgs\n",
        "\n",
        "# https://www.pakutaso.com/20180239038post-15116.html\n",
        "!wget https://www.pakutaso.com/shared/img/thumb/smIMGL4174_TP_V4.jpg\n",
        "\n",
        "%cd /content/OFA\n",
        "image = Image.open('/content/OFA/test_imgs/smIMGL4174_TP_V4.jpg')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZICM-fas3EB2"
      },
      "source": [
        "# Coordinate, Questions設定"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lfZ1RATH3iQY"
      },
      "outputs": [],
      "source": [
        "coords = \"522.0 396.0 595.0 469.0\" #@param {type:\"string\"}\n",
        "w, h = image.size\n",
        "w_resize_ratio = task.cfg.patch_image_size / w\n",
        "h_resize_ratio = task.cfg.patch_image_size / h\n",
        "bins = coord2bin(coords, w_resize_ratio, h_resize_ratio)\n",
        "question = \"What's in the region?\" #@param {type:\"string\"}\n",
        "instruction = \"\\\"\" + question + \" region: \\\" + bins\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 747
        },
        "id": "Nb7fFl_S4aCm",
        "outputId": "ce69be23-87ed-4225-cea4-a7142436e20f"
      },
      "outputs": [],
      "source": [
        "# Construct input sample & preprocess for GPU if cuda available\n",
        "sample = construct_sample(image, instruction)\n",
        "sample = utils.move_to_cuda(sample) if use_cuda else sample\n",
        "sample = utils.apply_to_sample(apply_half, sample) if use_fp16 else sample\n",
        "\n",
        "# Generate result\n",
        "with torch.no_grad():\n",
        "    hypos = task.inference_step(generator, models, sample)\n",
        "    tokens1, bins1, imgs1 = decode_fn(hypos[0][0][\"tokens\"], task.tgt_dict, task.bpe, generator)\n",
        "    tokens2, bins2, imgs2 = decode_fn(hypos[0][1][\"tokens\"], task.tgt_dict, task.bpe, generator)\n",
        "    tokens3, bins3, imgs3 = decode_fn(hypos[0][2][\"tokens\"], task.tgt_dict, task.bpe, generator)\n",
        "    tokens4, bins4, imgs4 = decode_fn(hypos[0][3][\"tokens\"], task.tgt_dict, task.bpe, generator)\n",
        "    tokens5, bins5, imgs5 = decode_fn(hypos[0][4][\"tokens\"], task.tgt_dict, task.bpe, generator)\n",
        "\n",
        "# display result\n",
        "img = cv2.cvtColor(numpy.asarray(image), cv2.COLOR_RGB2BGR)\n",
        "coord_list = bin2coord(bins, w_resize_ratio, h_resize_ratio)\n",
        "cv2.rectangle(\n",
        "    img,\n",
        "    (int(coord_list[0]), int(coord_list[1])),\n",
        "    (int(coord_list[2]), int(coord_list[3])),\n",
        "    (0, 255, 0),\n",
        "    3\n",
        ")\n",
        "cv2_imshow(img)\n",
        "\n",
        "print('Instruction: {}'.format(instruction))\n",
        "print('OFA\\'s Output1: {}, Probs: {}'.format(tokens1, hypos[0][0][\"score\"].exp().item()))\n",
        "print('OFA\\'s Output2: {}, Probs: {}'.format(tokens2, hypos[0][1][\"score\"].exp().item()))\n",
        "print('OFA\\'s Output3: {}, Probs: {}'.format(tokens3, hypos[0][2][\"score\"].exp().item()))\n",
        "print('OFA\\'s Output4: {}, Probs: {}'.format(tokens4, hypos[0][3][\"score\"].exp().item()))\n",
        "print('OFA\\'s Output5: {}, Probs: {}'.format(tokens5, hypos[0][4][\"score\"].exp().item()))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MPHiFngV5fEK"
      },
      "source": [
        "# Visual Grounding"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hFsRRCZq5iU-"
      },
      "source": [
        "## テスト画像取得"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Mj-S4qve5k_o",
        "outputId": "79e24dca-9336-497d-e6bf-d2a10c899e12"
      },
      "outputs": [],
      "source": [
        "%cd /content/OFA\n",
        "!mkdir test_imgs\n",
        "%cd test_imgs\n",
        "\n",
        "# https://www.pakutaso.com/20180239038post-15116.html\n",
        "!wget https://www.pakutaso.com/shared/img/thumb/smIMGL4174_TP_V4.jpg\n",
        "\n",
        "%cd /content/OFA\n",
        "image = Image.open('/content/OFA/test_imgs/smIMGL4174_TP_V4.jpg')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2PGJpp7o5nm7"
      },
      "source": [
        "## Question設定"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ms6u5eiv5un-"
      },
      "outputs": [],
      "source": [
        "question = \"Fruit that a woman has in her right hand\"  #@param {type:\"string\"}\n",
        "instruction = 'which region does the text \\\" ' + question + '\\\" describe?'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 643
        },
        "id": "TD4NMJry6IDT",
        "outputId": "ff98aaa4-7ab4-4f06-bcc1-96d76cb4837f"
      },
      "outputs": [],
      "source": [
        "# Construct input sample & preprocess for GPU if cuda available\n",
        "sample = construct_sample(image, instruction)\n",
        "sample = utils.move_to_cuda(sample) if use_cuda else sample\n",
        "sample = utils.apply_to_sample(apply_half, sample) if use_fp16 else sample\n",
        "\n",
        "# Generate result\n",
        "with torch.no_grad():\n",
        "    hypos = task.inference_step(generator, models, sample)\n",
        "    tokens, bins, imgs = decode_fn(hypos[0][0][\"tokens\"], task.tgt_dict, task.bpe, generator)\n",
        "\n",
        "# display result\n",
        "w_resize_ratio = task.cfg.patch_image_size / w\n",
        "h_resize_ratio = task.cfg.patch_image_size / h\n",
        "img = cv2.cvtColor(numpy.asarray(image), cv2.COLOR_RGB2BGR)\n",
        "coord_list = bin2coord(bins, w_resize_ratio, h_resize_ratio)\n",
        "cv2.rectangle(\n",
        "    img,\n",
        "    (int(coord_list[0]), int(coord_list[1])),\n",
        "    (int(coord_list[2]), int(coord_list[3])),\n",
        "    (0, 255, 0),\n",
        "    3\n",
        ")\n",
        "cv2_imshow(img)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "OFA_demo.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "orig_nbformat": 4
  },
  "nbformat": 4,
  "nbformat_minor": 0
}