Store vector embeddings for talks, and display related talks

.env.sample

@@ -1,4 +1,4 @@
 MEILI_MASTER_KEY=your local meili master key but in therory meilisearch can run without master key in development mode
 YOUTUBE_API_KEY=useful for scraping youtube videos but the app can run without it
 GITHUB_TOKEN=useful for the profile enhancement feature but the app can run without it
-
+OPENAI_ACCESS_TOKEN="change_me"

.gitignore

@@ -47,5 +47,6 @@
 
 /data.ms
 /data_tmp
+/meili_data
 
 /config/credentials/production.key

Gemfile

@@ -91,3 +91,4 @@ gem "meta-tags", "~> 2.18"
 gem "groupdate", "~> 6.2"
 
 gem "appsignal", "~> 3.4"
+gem "ruby-openai"

Gemfile.lock

@@ -144,6 +144,12 @@ GEM
     dry-cli (1.0.0)
     error_highlight (0.5.1)
     erubi (1.12.0)
+    faraday (2.7.7)
+      faraday-net_http (>= 2.0, < 3.1)
+      ruby2_keywords (>= 0.0.4)
+    faraday-multipart (1.0.4)
+      multipart-post (~> 2)
+    faraday-net_http (3.0.2)
     globalid (1.1.0)
       activesupport (>= 5.0)
     groupdate (6.2.1)
@@ -186,6 +192,7 @@ GEM
     minitest (5.18.0)
     msgpack (1.7.0)
     multi_xml (0.6.0)
+    multipart-post (2.3.0)
     net-http (0.3.2)
       uri
     net-imap (0.3.6)
@@ -264,7 +271,11 @@ GEM
       language_server-protocol (~> 3.17.0)
       sorbet-runtime
       syntax_tree (>= 6.1.1, < 7)
+    ruby-openai (4.1.0)
+      faraday (>= 1)
+      faraday-multipart (>= 1)
     ruby-progressbar (1.13.0)
+    ruby2_keywords (0.0.5)
     rubyzip (2.3.2)
     safely_block (0.4.0)
     selenium-webdriver (4.9.1)
@@ -360,6 +371,7 @@ DEPENDENCIES
   rails!
   redis (>= 4.0.1)
   ruby-lsp (~> 0.5.1)
+  ruby-openai
   selenium-webdriver
   sqlite3 (~> 1.4)
   standardrb (~> 1.0)

Procfile.dev

@@ -1,3 +1,3 @@
 web: bin/rails server -p 3000
 vite: bin/vite dev
-search: meilisearch
+search: docker rm -f rubyvideo-meilisearch && docker run --name rubyvideo-meilisearch -p 7700:7700 -v $(pwd)/meili_data:/meili_data getmeili/meilisearch:prototype-vector-store-1 meilisearch --env development --no-analytics --log-level=INFO

app/models/talk.rb

@@ -37,7 +37,7 @@ class Talk < ApplicationRecord
   delegate :name, to: :event, prefix: true, allow_nil: true
 
   # search
-  meilisearch do
+  meilisearch enqueue: true, raise_on_failure: Rails.env.development? do
     attribute :title
     attribute :description
     attribute :slug
@@ -49,14 +49,16 @@ class Talk < ApplicationRecord
     attribute :speakers do
       speakers.pluck(:name)
     end
-    searchable_attributes [:title, :description]
+    add_attribute :_vectors do
+      vectors
+    end
+    searchable_attributes [:title, :description, :_vector]
     sortable_attributes [:title]
+    filterable_attributes [:id]
 
     attributes_to_highlight ["*"]
   end
 
-  meilisearch enqueue: true
-
   def to_meta_tags
     {
       title: title,
@@ -83,4 +85,27 @@ def thumbnail_lg
   def thumbnail_xl
     self[:thumbnail_xl].presence || "https://i.ytimg.com/vi/#{video_id}/maxresdefault.jpg"
   end
+
+  def neighbors(limit: 5)
+    query_vector = Talk.index.document(self.id).fetch("_vectors", []).first
+    return Talk.none if query_vector.blank?
+    Talk.search("", vector: query_vector, limit: limit, filter: "id != #{self.id}")
+  end
+
+  def vectors
+    return nil unless ENV["OPENAI_ACCESS_TOKEN"].present?
+    # might need to split at some point if over the token limit (e.g. if including transcription)
+    @vectors ||= [self.class.embedding(title, description)]
+  end
+
+  def self.embedding(*inputs)
+    client = OpenAI::Client.new
+    response = client.embeddings(
+      parameters: {
+        model: "text-embedding-ada-002",
+        input: inputs.join("\n\n"),
+      },
+    )
+    response.dig("data", 0, "embedding")
+  end
 end

app/views/talks/_talk.html.erb

@@ -31,6 +31,16 @@
       <span><%#= talk.event_edition %></span>
     </p>
 
+    <% related_talks = talk.neighbors(limit: 5) %>
+    <% if related_talks.any? %>
+      <h2>Other talks you might be interested in</h2>
+      <ul class="list-inside list-disc">
+        <% related_talks.each do |related_talk| %>
+          <%= content_tag :li, link_to(related_talk.title.gsub(/\A:\s+/, ""), related_talk) %>
+        <% end %>
+      </ul>
+    <% end %>
+
     <% if action_name != "show" %>
       <%= link_to "Show this talk", talk, class: "rounded-lg py-3 px-5 bg-gray-100 inline-block font-medium" %>
       <%= link_to 'Edit this talk', edit_talk_path(talk), class: "rounded-lg py-3 ml-2 px-5 bg-gray-100 inline-block font-medium" %>

config/initializers/meilisearch.rb

@@ -1,4 +1,4 @@
 MeiliSearch::Rails.configuration = {
   meilisearch_url: Rails.env.local? ? "http://localhost:7700" : "http://91.107.208.207:7700", # example: http://localhost:7700
-  meilisearch_api_key: ENV["MEILI_MASTER_KEY"]
+  meilisearch_api_key: Rails.env.local? ? nil : ENV["MEILI_MASTER_KEY"]
 }

config/initializers/openai.rb

@@ -0,0 +1,5 @@
+if ENV["OPENAI_ACCESS_TOKEN"].present?
+  OpenAI.configure do |config|
+    config.access_token = ENV.fetch("OPENAI_ACCESS_TOKEN")
+  end
+end