Merge pull request #142 from leouofa/add_invalid_json_to_articles

adding invalid json tag to articles and refactoring the rewrite job

app/jobs/articles/rewrite_article_job.rb

@@ -86,14 +86,14 @@ def perform(article:)
 
       Rails.logger.debug "invalid_json: #{invalid_json} | counter: #{counter}"
 
-      sleep(20)
+      sleep(25)
     end
 
     unless response["error"].present? || invalid_json
       parsed_response = JSON.parse(extracted_json_response)
-
       article.update(rewritten_text: parsed_response["content"])
-
+    else
+      article.update(invalid_json:)
     end
 
     sleep(10)

app/models/article.rb

@@ -13,6 +13,7 @@
 #  created_at       :datetime         not null
 #  updated_at       :datetime         not null
 #  embedding        :vector(768)
+#  invalid_json     :boolean          default(FALSE)
 #
 class Article < ApplicationRecord
   belongs_to :pillar_column
@@ -42,6 +43,7 @@ class Article < ApplicationRecord
   scope :with_three_links_without_rewritten_text, lambda {
     with_three_links
       .where(rewritten_text: [nil, ''])
+      .where(invalid_json: false)
   }
 
   private

db/migrate/20240703164512_add_invalid_json_to_articles.rb

@@ -0,0 +1,5 @@
+class AddInvalidJsonToArticles < ActiveRecord::Migration[7.0]
+  def change
+    add_column :articles, :invalid_json, :boolean, default: false
+  end
+end

spec/factories/articles.rb

@@ -13,6 +13,7 @@
 #  created_at       :datetime         not null
 #  updated_at       :datetime         not null
 #  embedding        :vector(768)
+#  invalid_json     :boolean          default(FALSE)
 #
 FactoryBot.define do
   factory :article do