CWS_PResUNet_demo.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4kn5RKIUa-hI"
      },
      "source": [
        "論文  \n",
        "https://arxiv.org/abs/2112.04685<br>\n",
        "<br>  \n",
        "GitHub  \n",
        "https://github.com/haoheliu/2021-ISMIR-MSS-Challenge-CWS-PResUNet<br>\n",
        "<br>\n",
        "<a href=\"https://colab.research.google.com/github/kaz12tech/ai_demos/blob/master/CWS_PResUNet_demo.ipynb\" target=\"_blank\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AI_4j9-Ga-hM"
      },
      "source": [
        "## 環境セットアップ"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9eYPCkxTa-hM"
      },
      "source": [
        "## GPU確認"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2PcII5fLdHnJ"
      },
      "outputs": [],
      "source": [
        "!nvidia-smi"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fWDnTPp4a-hO"
      },
      "source": [
        "## GitHubからコード取得"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "R-Y5auuvnnnT"
      },
      "outputs": [],
      "source": [
        "%cd /content\n",
        "\n",
        "!git clone https://github.com/haoheliu/2021-ISMIR-MSS-Challenge-CWS-PResUNet.git"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qW0Ced2ia-hO"
      },
      "source": [
        "## ライブラリのインストール"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LScsvO3UppqT"
      },
      "outputs": [],
      "source": [
        "%cd /content/2021-ISMIR-MSS-Challenge-CWS-PResUNet\n",
        "\n",
        "!pip install --upgrade gdown\n",
        "\n",
        "! pip3 install -r requirements.txt\n",
        "# ModuleNotFoundError: No module named 'torchtext.legacy' 回避のため torchtextはバージョン固定\n",
        "! pip3 install torchtext==0.8.1 demucs==3.0.1"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nju6Dheka-hU"
      },
      "source": [
        "# Vocal抽出(テスト音源)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eSjzocGWT1w1"
      },
      "outputs": [],
      "source": [
        "%cd /content/2021-ISMIR-MSS-Challenge-CWS-PResUNet\n",
        "\n",
        "!python3 main.py -i example/test -o example/results -s vocals other --cuda "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-leEsLDOcBy4"
      },
      "source": [
        "# 抽出結果再生"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WKmtaJjZcHFX"
      },
      "outputs": [],
      "source": [
        "import IPython.display as ipd\n",
        "ipd.Audio(\"example/test/xuemaojiao.wav\", rate=44100)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DeOzv_nccBG0"
      },
      "outputs": [],
      "source": [
        "ipd.Audio(\"example/results/xuemaojiao/vocals.wav\", rate=44100)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gj_Wi_1Xi-1Y"
      },
      "source": [
        "# Vocal抽出(Youtube)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n9ga_HvQni43"
      },
      "source": [
        "## 動画のセットアップ"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bZIpkwZclLl_"
      },
      "outputs": [],
      "source": [
        "!pip install yt-dlp moviepy "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6xvryU3UjswO"
      },
      "outputs": [],
      "source": [
        "from yt_dlp import YoutubeDL\n",
        "\n",
        "from moviepy.video.fx.resize import resize\n",
        "from moviepy.editor import VideoFileClip, AudioFileClip, ImageSequenceClip, CompositeAudioClip\n",
        "from moviepy.video.io.ffmpeg_tools import ffmpeg_extract_subclip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oR2sPZlbjDWK"
      },
      "outputs": [],
      "source": [
        "video_url = 'https://www.youtube.com/watch?v=Qd01-6xVSHk' #@param {type:\"string\"}\n",
        "\n",
        "#@markdown 動画の切り抜き範囲(秒)を指定してください。\\\n",
        "#@markdown 30秒以上の場合OOM発生の可能性が高いため注意\n",
        "start_sec =  57#@param {type:\"integer\"}\n",
        "end_sec =  69#@param {type:\"integer\"}\n",
        "\n",
        "(start_pt, end_pt) = (start_sec, end_sec)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EiWMvpR2jhuk"
      },
      "source": [
        "## 動画取得"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3qjBzFL2jhFW"
      },
      "outputs": [],
      "source": [
        "!mkdir -p /content/2021-ISMIR-MSS-Challenge-CWS-PResUNet/videos/results\n",
        "\n",
        "\n",
        "download_resolution = 360\n",
        "full_video_path = '/content/2021-ISMIR-MSS-Challenge-CWS-PResUNet/videos/full_video.mp4'\n",
        "input_clip_path = '/content/2021-ISMIR-MSS-Challenge-CWS-PResUNet/videos/input_clip.mp4'\n",
        "input_audio_path = '/content/2021-ISMIR-MSS-Challenge-CWS-PResUNet/videos/input_clip.wav'\n",
        "\n",
        "# 動画ダウンロード\n",
        "ydl_opts = {'format': f'best[height<={download_resolution}]', 'overwrites': True, 'outtmpl': full_video_path}\n",
        "with YoutubeDL(ydl_opts) as ydl:\n",
        "    ydl.download([video_url])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7_1dsxa0nmiO"
      },
      "source": [
        "## 指定区間切り抜き"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GuaYhYQVjnMw"
      },
      "outputs": [],
      "source": [
        "# extract the relevant subclip:\n",
        "with VideoFileClip(full_video_path) as video:\n",
        "    subclip = video.subclip(start_pt, end_pt)\n",
        "    subclip.write_videofile(input_clip_path)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pj4pOpFroWrm"
      },
      "source": [
        "## 切り抜き箇所(0~5秒)再生"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "N8ylnfbMnw-7"
      },
      "outputs": [],
      "source": [
        "clip = VideoFileClip(input_clip_path)\n",
        "\n",
        "if 5 < (end_pt-start_pt):\n",
        "  clip = clip.subclip(0, 5)\n",
        "  \n",
        "clip = resize(clip, height=420)\n",
        "clip.ipython_display()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JJbRDcnnpS19"
      },
      "source": [
        "## audioファイル取得"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PgA1-74TpSI7"
      },
      "outputs": [],
      "source": [
        "clip = VideoFileClip(input_clip_path)\n",
        "clip.audio.write_audiofile(input_audio_path, codec='pcm_s16le')\n",
        "\n",
        "ipd.Audio(input_audio_path, rate=44100)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uH7KnqufsqMq"
      },
      "source": [
        "## Vocal抽出"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KfuqfVs1ssPo"
      },
      "outputs": [],
      "source": [
        "!python3 main.py -i {input_audio_path} -o /content/2021-ISMIR-MSS-Challenge-CWS-PResUNet/videos/results -s vocals other --cuda "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3rG40N5tuEAv"
      },
      "source": [
        "### Vocal"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "EA9v4NLytROL"
      },
      "outputs": [],
      "source": [
        "ipd.Audio('/content/2021-ISMIR-MSS-Challenge-CWS-PResUNet/videos/results/input_clip/vocals.wav', rate=44100)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vzxgkjz8uFZZ"
      },
      "source": [
        "### Vocal以外"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SMk5v11Vtc8B"
      },
      "outputs": [],
      "source": [
        "ipd.Audio('/content/2021-ISMIR-MSS-Challenge-CWS-PResUNet/videos/results/input_clip/other.wav', rate=44100)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ds9ocFAFuITI"
      },
      "source": [
        "## Vocalのみの動画を作成"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "J77ugzJmwTIJ"
      },
      "outputs": [],
      "source": [
        "videoclip = VideoFileClip(input_clip_path)\n",
        "audioclip = AudioFileClip(\"/content/2021-ISMIR-MSS-Challenge-CWS-PResUNet/videos/results/input_clip/vocals.wav\")\n",
        "\n",
        "new_audioclip = CompositeAudioClip([audioclip])\n",
        "videoclip.audio = new_audioclip\n",
        "videoclip.write_videofile(\"/content/2021-ISMIR-MSS-Challenge-CWS-PResUNet/videos/results/input_clip/vocals.mp4\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hUSYBKwzu6Um"
      },
      "outputs": [],
      "source": [
        "clip = VideoFileClip('/content/2021-ISMIR-MSS-Challenge-CWS-PResUNet/videos/results/input_clip/vocals.mp4')\n",
        "  \n",
        "clip = resize(clip, height=420)\n",
        "clip.ipython_display()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "CWS-PResUNet_demo.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "orig_nbformat": 4
  },
  "nbformat": 4,
  "nbformat_minor": 0
}