Merge pull request #5797 from avalonmediasystem/fedora_solr_errors

Add option to rescue Solr/Fedora connection errors
avalonmediasystem · May 10, 2024 · 29366ae · 29366ae
2 parents eb8e7d4 + 1760915
commit 29366ae
Show file tree

Hide file tree

Showing 7 changed files with 213 additions and 0 deletions.
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
@@ -34,6 +34,11 @@ class ApplicationController < ActionController::Base
   before_action :set_no_cache_headers, if: proc{|c| request.xhr? }
   prepend_before_action :remove_zero_width_chars
 
+  rescue_from RSolr::Error::ConnectionRefused, :with => :handle_solr_connection_error
+  rescue_from RSolr::Error::Timeout, :with => :handle_solr_connection_error
+  rescue_from Blacklight::Exceptions::ECONNREFUSED, :with => :handle_solr_connection_error
+  rescue_from Faraday::ConnectionFailed, :with => :handle_fedora_connection_error
+
   def set_no_cache_headers
     response.headers["Cache-Control"] = "no-cache, no-store"
     response.headers["Pragma"] = "no-cache"
@@ -245,4 +250,26 @@ def strip_zero_width_chars!(obj)
     def should_store_return_url?
       !(request.xhr? || request.format != "html" || request.path.start_with?("/users/") || request.path.end_with?("poster") || request.path.end_with?("thumbnail"))
     end
+
+    def handle_solr_connection_error(exception)
+      raise if Settings.app_controller.solr_and_fedora.raise_on_connection_error
+      Rails.logger.error(exception.class.to_s + ': ' + exception.message + '\n' + exception.backtrace.join('\n'))
+
+      if request.format == :json
+        render json: {errors: [exception.message]}, status: 503
+      else
+        render '/errors/solr_connection', layout: false, status: 503
+      end
+    end
+
+    def handle_fedora_connection_error(exception)
+      raise if Settings.app_controller.solr_and_fedora.raise_on_connection_error
+      Rails.logger.error(exception.class.to_s + ': ' + exception.message + '\n' + exception.backtrace.join('\n'))
+
+      if request.format == :json
+        render json: {errors: [exception.message]}, status: 503
+      else
+        render '/errors/fedora_connection', status: 503
+      end
+    end
 end
diff --git a/app/jobs/application_job.rb b/app/jobs/application_job.rb
@@ -13,7 +13,18 @@
 # ---  END LICENSE_HEADER BLOCK  ---
 
 class ApplicationJob < ActiveJob::Base
+  rescue_from RSolr::Error::ConnectionRefused, :with => :handle_connection_error
+  rescue_from RSolr::Error::Timeout, :with => :handle_connection_error
+  rescue_from Blacklight::Exceptions::ECONNREFUSED, :with => :handle_connection_error
+  rescue_from Faraday::ConnectionFailed, :with => :handle_connection_error
+
   rescue_from Ldp::Gone do |exception|
     Rails.logger.error(exception.message + '\n' + exception.backtrace.join('\n'))
   end
+
+  private
+    def handle_connection_error(exception)
+      raise if Settings.app_job.solr_and_fedora.raise_on_connection_error
+      Rails.logger.error(exception.class.to_s + ': ' + exception.message + '\n' + exception.backtrace.join('\n'))
+    end
 end
diff --git a/app/views/errors/fedora_connection.html.erb b/app/views/errors/fedora_connection.html.erb
@@ -0,0 +1,22 @@
+<%#
+Copyright 2011-2024, The Trustees of Indiana University and Northwestern
+  University.  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed
+  under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+  CONDITIONS OF ANY KIND, either express or implied. See the License for the
+  specific language governing permissions and limitations under the License.
+---  END LICENSE_HEADER BLOCK  ---
+%>
+<div id="main_content_container" class="container-fluid">
+  <div class="content" class="col-md-8">
+    <h2>Fedora Connection Error</h2>
+    <p>The application was unable to connect to the Fedora database. Please try again in a few minutes.</p>
+    <p>If the problem persists contact your support staff.</p>
+  </div>
+</div>
diff --git a/app/views/errors/solr_connection.html.erb b/app/views/errors/solr_connection.html.erb
@@ -0,0 +1,70 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Solr Connection Error (503)</title>
+  <meta name="viewport" content="width=device-width,initial-scale=1">
+  <style>
+  body {
+    background-color: #EFEFEF;
+    color: #2E2F30;
+    text-align: center;
+    font-family: arial, sans-serif;
+    margin: 0;
+  }
+
+  div.dialog {
+    width: 95%;
+    max-width: 33em;
+    margin: 4em auto 0;
+  }
+
+  div.dialog > div {
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #BBB;
+    border-top: #B00100 solid 4px;
+    border-top-left-radius: 9px;
+    border-top-right-radius: 9px;
+    background-color: white;
+    padding: 7px 12% 0;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+
+  h1 {
+    font-size: 100%;
+    color: #730E15;
+    line-height: 1.5em;
+  }
+
+  div.dialog > p {
+    margin: 0 0 1em;
+    padding: 1em;
+    background-color: #F7F7F7;
+    border: 1px solid #CCC;
+    border-right-color: #999;
+    border-left-color: #999;
+    border-bottom-color: #999;
+    border-bottom-left-radius: 4px;
+    border-bottom-right-radius: 4px;
+    border-top-color: #DADADA;
+    color: #666;
+    box-shadow: 0 3px 8px rgba(50, 50, 50, 0.17);
+  }
+  </style>
+</head>
+
+<body>
+  <div class="dialog">
+    <div>
+      <h1>Solr Connection Error</h1>
+    </div>
+    <p>
+      The application was unable to connect to the Solr database. 
+      </br>Please try again in a few minutes.
+      </br>
+      </br>If the problem persists contact your support staff.
+    </p>
+  </div>
+</body>
+</html>
diff --git a/config/settings.yml b/config/settings.yml
@@ -44,6 +44,12 @@ solr:
     rule:
     shards:
     snitch:
+app_controller:
+  solr_and_fedora:
+    raise_on_connection_error: false
+app_job:
+  solr_and_fedora:
+    raise_on_connection_error: true
 zookeeper:
   connection_str: "localhost:9983/configs"
 streaming:

diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb
@@ -80,6 +80,50 @@ def show
       get :show, params: { id: 'deleted-id' }
       expect(response).to render_template("errors/deleted_pid")
     end
+
+    context 'raise_on_connection_error disabled' do
+      let(:request_context) { { body: "request_context" } }
+      let(:e) { { body: "error_response" } }
+
+      before :each do
+        allow(Settings.app_controller.solr_and_fedora).to receive(:raise_on_connection_error).and_return(false)
+      end
+
+      [RSolr::Error::ConnectionRefused, RSolr::Error::Timeout, Blacklight::Exceptions::ECONNREFUSED, Faraday::ConnectionFailed].each do |error_code|
+        it "rescues #{error_code} errors" do
+          raised_error = error_code == RSolr::Error::Timeout ? error_code.new(request_context, e) : error_code      
+          allow(controller).to receive(:show).and_raise(raised_error)
+          allow_any_instance_of(Exception).to receive(:backtrace).and_return(["Test trace"])
+          allow_any_instance_of(Exception).to receive(:message).and_return('Connection reset by peer')
+          expect(Rails.logger).to receive(:error).with(error_code.to_s + ': Connection reset by peer\nTest trace')
+          expect { get :show, params: { id: 'abc1234' } }.to_not raise_error
+        end
+
+        it "renders error template for #{error_code} errors" do
+          error_template = error_code == Faraday::ConnectionFailed ? 'errors/fedora_connection' : 'errors/solr_connection'
+          raised_error = error_code == RSolr::Error::Timeout ? error_code.new(request_context, e) : error_code
+          allow(controller).to receive(:show).and_raise(raised_error)
+          get :show, params: { id: 'abc1234' }
+          expect(response).to render_template(error_template)
+        end
+      end
+    end
+
+    context 'raise_on_connection_error enabled' do
+      let(:request_context) { { body: "request_context" } }
+      let(:e) { { body: "error_response" } }
+
+      [RSolr::Error::ConnectionRefused, RSolr::Error::Timeout, Blacklight::Exceptions::ECONNREFUSED, Faraday::ConnectionFailed].each do |error_code|
+        it "raises #{error_code} errors" do
+          raised_error = error_code == RSolr::Error::Timeout ? error_code.new(request_context, e) : error_code 
+          allow(Settings.app_controller.solr_and_fedora).to receive(:raise_on_connection_error).and_return(true)
+          allow(controller).to receive(:show).and_raise(raised_error)
+          allow_any_instance_of(Exception).to receive(:backtrace).and_return(["Test trace"])
+          allow_any_instance_of(Exception).to receive(:message).and_return('Connection reset by peer')
+          expect { get :show, params: { id: 'abc1234' } }.to raise_error(error_code, 'Connection reset by peer')
+        end
+      end
+    end
   end
 
   describe "rewrite_v4_ids" do

diff --git a/spec/jobs/application_job_spec.rb b/spec/jobs/application_job_spec.rb
@@ -22,5 +22,38 @@
       expect(Rails.logger).to receive(:error).with('Ldp::Gone\nTest trace')
       expect { described_class.perform_now }.to_not raise_error
     end
+
+    context 'raise_on_connection_error disabled' do
+      let(:request_context) { { body: "request_context" } }
+      let(:e) { { body: "error_response" } }
+
+      [RSolr::Error::ConnectionRefused, RSolr::Error::Timeout, Blacklight::Exceptions::ECONNREFUSED, Faraday::ConnectionFailed].each do |error_code|
+        it "rescues #{error_code} errors" do
+          raised_error = error_code == RSolr::Error::Timeout ? error_code.new(request_context, e) : error_code 
+          allow(Settings.app_job.solr_and_fedora).to receive(:raise_on_connection_error).and_return(false)
+          allow_any_instance_of(described_class).to receive(:perform).and_raise(raised_error)
+          allow_any_instance_of(Exception).to receive(:backtrace).and_return(["Test trace"])
+          allow_any_instance_of(Exception).to receive(:message).and_return('Connection reset by peer')
+          expect(Rails.logger).to receive(:error).with(error_code.to_s + ': Connection reset by peer\nTest trace')
+          expect { described_class.perform_now }.to_not raise_error
+        end
+      end
+    end
+
+    context 'raise_on_connection_error enabled' do
+      let(:request_context) { { body: "request_context" } }
+      let(:e) { { body: "error_response" } }
+
+      [RSolr::Error::ConnectionRefused, RSolr::Error::Timeout, Blacklight::Exceptions::ECONNREFUSED, Faraday::ConnectionFailed].each do |error_code|
+        it "raises #{error_code} errors" do
+          raised_error = error_code == RSolr::Error::Timeout ? error_code.new(request_context, e) : error_code 
+          allow(Settings.app_job.solr_and_fedora).to receive(:raise_on_connection_error).and_return(true)
+          allow_any_instance_of(described_class).to receive(:perform).and_raise(raised_error)
+          allow_any_instance_of(Exception).to receive(:backtrace).and_return(["Test trace"])
+          allow_any_instance_of(Exception).to receive(:message).and_return('Connection reset by peer')
+          expect { described_class.perform_now }.to raise_error(error_code, 'Connection reset by peer')
+        end
+      end
+    end
   end
 end