Add Transcript to videos (#80)

* update deps and Vite

* remove warning message

* adds the ability to fetch transcript from youtube

* refactor transcript serializer

* basic display of the transcript

* update vlitejs

* add seekTo functionality

* extract to partial

* add transcript to a tab

* add transcript to the search

* lint

* fix vlite imports for V6

* return empty transcript when missing

.github/workflows/ci.yml

@@ -1,12 +1,6 @@
 name: linters
 
-on:
-  pull_request:
-    branches:
-      - "*"
-  push:
-    branches:
-      - main
+on: [push]
 
 concurrency: ci-${{ github.ref }}
 
@@ -43,6 +37,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       RAILS_ENV: test
+      MEILI_MASTER_KEY: masterKey
     steps:
       - uses: actions/checkout@v4
 
@@ -61,6 +56,9 @@ jobs:
       - name: Install dependencies
         run: yarn install --frozen-lockfile
 
+      - name: Meilisearch setup with Docker
+        run: docker run -d -p 7700:7700 getmeili/meilisearch:v1.1 meilisearch --master-key=masterKey --no-analytics
+
       - name: Build assets
         run: bin/vite build --clear --mode=test
 

Gemfile

@@ -112,3 +112,7 @@ gem "view_component", "~> 3.7"
 gem "dry-initializer-rails"
 
 gem "dry-types", "~> 1.7"
+
+gem "google-protobuf", require: false
+
+gem "active_job-performs", "~> 0.3.1"

Gemfile.lock

@@ -50,6 +50,8 @@ GEM
       erubi (~> 1.11)
       rails-dom-testing (~> 2.2)
       rails-html-sanitizer (~> 1.6)
+    active_job-performs (0.3.1)
+      activejob (>= 6.1)
     activejob (7.1.3.4)
       activesupport (= 7.1.3.4)
       globalid (>= 0.3.6)
@@ -170,6 +172,18 @@ GEM
       raabro (~> 1.4)
     globalid (1.2.1)
       activesupport (>= 6.1)
+    google-protobuf (4.27.1-aarch64-linux)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.27.1-arm64-darwin)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.27.1-x86_64-darwin)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.27.1-x86_64-linux)
+      bigdecimal
+      rake (>= 13)
     groupdate (6.4.0)
       activesupport (>= 6.1)
     hashdiff (1.1.0)
@@ -182,7 +196,7 @@ GEM
       activesupport (>= 3.0)
       nokogiri (>= 1.6)
     io-console (0.7.2)
-    irb (1.13.1)
+    irb (1.14.0)
       rdoc (>= 4.0.0)
       reline (>= 0.4.2)
     jbuilder (2.12.0)
@@ -223,7 +237,7 @@ GEM
       actionpack (>= 6.0.0, < 7.2)
     method_source (1.1.0)
     mini_mime (1.1.5)
-    minitest (5.23.1)
+    minitest (5.24.1)
     msgpack (1.7.2)
     multi_xml (0.7.1)
       bigdecimal (~> 3.1)
@@ -266,7 +280,7 @@ GEM
       nio4r (~> 2.0)
     raabro (1.4.0)
     racc (1.8.0)
-    rack (3.1.3)
+    rack (3.1.6)
     rack-mini-profiler (3.3.1)
       rack (>= 1.2.0)
     rack-proxy (0.7.7)
@@ -429,7 +443,7 @@ GEM
     websocket-extensions (0.1.5)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.6.15)
+    zeitwerk (2.6.16)
 
 PLATFORMS
   aarch64-linux
@@ -439,6 +453,7 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
+  active_job-performs (~> 0.3.1)
   activerecord-enhancedsqlite3-adapter
   ahoy_matey (~> 4.2)
   annotate
@@ -456,6 +471,7 @@ DEPENDENCIES
   dry-types (~> 1.7)
   erb_lint (~> 0.4.0)
   error_highlight (>= 0.4.0)
+  google-protobuf
   groupdate (~> 6.2)
   inline_svg (~> 1.9)
   jbuilder

app/assets/stylesheets/application.css

@@ -8,3 +8,5 @@
 @import "components/transition.css";
 @import "components/typography.css";
 @import "components/video.css";
+
+@import "vlitejs/vlite.css";

app/clients/youtube/transcript.rb

@@ -0,0 +1,47 @@
+require "message_pb"
+
+module Youtube
+  class Transcript
+    attr_reader :response
+
+    def get(video_id)
+      message = {one: "asr", two: "en"}
+      typedef = MessageType
+      two = get_base64_protobuf(message, typedef)
+
+      message = {one: video_id, two: two}
+      params = get_base64_protobuf(message, typedef)
+
+      url = "https://www.youtube.com/youtubei/v1/get_transcript"
+      headers = {"Content-Type" => "application/json"}
+      body = {
+        context: {
+          client: {
+            clientName: "WEB",
+            clientVersion: "2.20240313"
+          }
+        },
+        params: params
+      }
+
+      @response = HTTParty.post(url, headers: headers, body: body.to_json)
+      JSON.parse(@response.body)
+    end
+
+    def self.get(video_id)
+      new.get(video_id)
+    end
+
+    private
+
+    def encode_message(message, typedef)
+      encoded_message = typedef.new(message)
+      encoded_message.to_proto
+    end
+
+    def get_base64_protobuf(message, typedef)
+      encoded_data = encode_message(message, typedef)
+      Base64.encode64(encoded_data).delete("\n")
+    end
+  end
+end

app/javascript/controllers/video_player_controller.js

@@ -1,7 +1,6 @@
 import { Controller } from '@hotwired/stimulus'
-import 'vlitejs/dist/vlite.css'
 import Vlitejs from 'vlitejs'
-import VlitejsYoutube from 'vlitejs/dist/providers/youtube'
+import VlitejsYoutube from 'vlitejs/providers/youtube.js'
 
 Vlitejs.registerProvider('youtube', VlitejsYoutube)
 
@@ -28,6 +27,8 @@ export default class extends Controller {
       const playbackRateSelect = this.createPlaybackRateSelect(this.playbackRateOptions, player)
       volumeButton.parentNode.insertBefore(playbackRateSelect, volumeButton.nextSibling)
     }
+    // for seekTo to work we need to store again the player instance
+    this.player = player
   }
 
   createPlaybackRateSelect (options, player) {
@@ -46,4 +47,12 @@ export default class extends Controller {
 
     return playbackRateSelect
   }
+
+  seekTo (event) {
+    const { time } = event.params
+
+    if (time) {
+      this.player.seekTo(time)
+    }
+  }
 }

app/models/cue.rb

@@ -0,0 +1,33 @@
+class Cue
+  attr_reader :start_time, :end_time, :text
+
+  def initialize(start_time, end_time, text)
+    @start_time = start_time
+    @end_time = end_time
+    @text = text
+  end
+
+  def to_s
+    "#{start_time} --> #{end_time}\n#{text}"
+  end
+
+  def to_h
+    {
+      start_time: start_time,
+      end_time: end_time,
+      text: text
+    }
+  end
+
+  def start_time_in_seconds
+    time_string_to_seconds(start_time)
+  end
+
+  def time_string_to_seconds(time_string)
+    parts = time_string.split(":").map(&:to_f)
+    hours = parts[0] * 3600
+    minutes = parts[1] * 60
+    seconds = parts[2]
+    (hours + minutes + seconds).to_i
+  end
+end

app/models/talk.rb

@@ -24,9 +24,12 @@
 #
 # rubocop:enable Layout/LineLength
 class Talk < ApplicationRecord
+  extend ActiveJob::Performs
   include Sluggable
   include Suggestable
   slug_from :title
+
+  # include MeiliSearch
   include MeiliSearch::Rails
   ActiveRecord_Relation.include Pagy::Meilisearch
   extend Pagy::Meilisearch
@@ -36,12 +39,19 @@ class Talk < ApplicationRecord
   has_many :speaker_talks, dependent: :destroy, inverse_of: :talk, foreign_key: :talk_id
   has_many :speakers, through: :speaker_talks
 
+  serialize :transcript, coder: TranscriptSerializer
+
   # validations
   validates :title, presence: true
 
   # delegates
   delegate :name, to: :event, prefix: true, allow_nil: true
 
+  # jobs
+  performs :update_transcript!, queue_as: :low do
+    retry_on StandardError, wait: :polynomially_longer
+  end
+
   # search
   meilisearch do
     attribute :title
@@ -58,7 +68,10 @@ class Talk < ApplicationRecord
     attribute :event_name do
       event_name
     end
-    searchable_attributes [:title, :description, :speaker_names, :event_name]
+    attribute :transcript do
+      transcript.to_text
+    end
+    searchable_attributes [:title, :description, :speaker_names, :event_name, :transcript]
     sortable_attributes [:title]
 
     attributes_to_highlight ["*"]
@@ -118,4 +131,9 @@ def thumbnail_xl
   def related_talks(limit: 6)
     Talk.order("RANDOM()").excluding(self).limit(limit)
   end
+
+  def update_transcript!
+    youtube_transcript = Youtube::Transcript.get(video_id)
+    update!(transcript: Transcript.create_from_youtube_transcript(youtube_transcript))
+  end
 end

app/models/transcript.rb

@@ -0,0 +1,65 @@
+class Transcript
+  include Enumerable
+
+  attr_reader :cues
+
+  def initialize
+    @cues = []
+  end
+
+  def add_cue(cue)
+    @cues << cue
+  end
+
+  def to_h
+    @cues.map { |cue| cue.to_h }
+  end
+
+  def to_json
+    to_h.to_json
+  end
+
+  def to_text
+    @cues.map { |cue| cue.text }.join("\n\n")
+  end
+
+  def to_vtt
+    vtt_content = "WEBVTT\n\n"
+    @cues.each_with_index do |cue, index|
+      vtt_content += "#{index + 1}\n"
+      vtt_content += "#{cue}\n\n"
+    end
+    vtt_content
+  end
+
+  def each(&)
+    @cues.each(&)
+  end
+
+  class << self
+    def create_from_youtube_transcript(youtube_transcript)
+      transcript = Transcript.new
+      events = youtube_transcript.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer", "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer", "initialSegments")
+      if events
+        events.each do |event|
+          segment = event["transcriptSegmentRenderer"]
+          start_time = format_time(segment["startMs"].to_i)
+          end_time = format_time(segment["endMs"].to_i)
+          text = segment.dig("snippet", "runs")&.map { |run| run["text"] }&.join || ""
+          transcript.add_cue(Cue.new(start_time, end_time, text))
+        end
+      else
+        transcript.add_cue(Cue.new("00:00:00.000", "00:00:00.000", "NOTE No transcript data available"))
+      end
+      transcript
+    end
+
+    def format_time(ms)
+      hours = ms / (1000 * 60 * 60)
+      minutes = (ms % (1000 * 60 * 60)) / (1000 * 60)
+      seconds = (ms % (1000 * 60)) / 1000
+      milliseconds = ms % 1000
+      format("%02d:%02d:%02d.%03d", hours, minutes, seconds, milliseconds)
+    end
+  end
+end

app/serializers/transcript_serializer.rb

@@ -0,0 +1,16 @@
+class TranscriptSerializer
+  def self.dump(transcript)
+    transcript.to_json
+  end
+
+  def self.load(transcript_json)
+    transcript = Transcript.new
+    return transcript if transcript_json.nil? || transcript_json.empty?
+
+    cues_array = JSON.parse(transcript_json, symbolize_names: true)
+    cues_array.each do |cue_hash|
+      transcript.add_cue(Cue.new(cue_hash[:start_time], cue_hash[:end_time], cue_hash[:text]))
+    end
+    transcript
+  end
+end