Merge pull request #420 from openstax/raise-columns

RAISE columns

Author: Dantemss

.github/workflows/assets.yml

@@ -12,7 +12,7 @@ on:
 jobs:
   assets:
     timeout-minutes: 30
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/migrations.yml

@@ -14,7 +14,7 @@ on:
 jobs:
   migrations:
     timeout-minutes: 30
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     services:
       postgres:
         image: postgres:11

.github/workflows/tests.yml

@@ -14,7 +14,7 @@ on:
 jobs:
   tests:
     timeout-minutes: 30
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     services:
       postgres:
         image: postgres:11

app/routines/exercises/import/assessments.rb

@@ -1,14 +1,5 @@
 # Imports exercies from a spreadsheet for Assessments
-# The first row contains column headers. Required columns:
-# UUID (page UUID)
-# Pre or Post
-# Question Stem
-# Answer Choice A
-# Answer Choice B
-# Answer Choice C
-# Answer Choice D
-# Correct Answer (A, B, C or D)
-# Detailed Solution
+# The first row contains column headers. The only required column is Question Stem.
 module Exercises
   module Import
     class Assessments
@@ -25,79 +16,170 @@ def exec(filename:, book_uuid:)
         author = User.find(AUTHOR_ID)
         copyright_holder = User.find(COPYRIGHT_HOLDER_ID) rescue author # So it works in the dev env with only 1 user
 
+        initialized = false
+
+        question_stem_index = nil
         uuid_index = nil
+        section_index = nil
+
+        page_uuid_by_book_location = {}
+
+        nickname_index = nil
+
         pre_or_post_index = nil
-        question_stem_index = nil
+
+        teks_index = nil
+        machine_teks_index = nil
+
+        raise_id_index = nil
+
+        background_index = nil
+        multi_step_index = nil
+        block_index = nil
+
         answer_choice_indices = nil
         correct_answer_index = nil
+        feedback_indices = nil
+
         detailed_solution_index = nil
+
+        row_number = nil
+
+        multi_step = nil
+        exercise = nil
         record_failures do |failures|
-          ProcessSpreadsheet.call(filename: filename, headers: :downcase) do |headers, row, row_index|
-            uuid_index ||= headers.index { |header| header == 'uuid' || header == 'page uuid' }
+          save = ->(exercise, row_number) do
+            return if exercise.nil? || row_number.nil?
 
-            section_index ||= headers.index { |header| header == 'section' }
-            raise ArgumentError, 'Could not find "UUID" or "Section" columns' if uuid_index.nil? && section_index.nil?
+            begin
+              exercise.save!
+              exercise.publication.publish.save!
 
-            unless section_index.nil?
-              book = OpenStax::Content::Abl.new.approved_books.find { |book| book.uuid == book_uuid }
-              page_uuid_by_book_location = {}
-              book.all_pages.each { |page| page_uuid_by_book_location[page.book_location] = page.uuid }
-              raise ArgumentError, "Could not find book with UUID #{book_uuid} in the ABL" if book.nil?
+              Rails.logger.info { "Imported row ##{row_number} - New exercise ID: #{exercise.uid}" }
+            rescue StandardError => error
+              Rails.logger.error { "Failed to import row ##{row_number} - #{error.message}" }
+              failures[row_number] = error.to_s
             end
+          end
 
-            pre_or_post_index ||= headers.index { |header| header&.start_with?('pre') && header.end_with?('post') }
-            raise ArgumentError, 'Could not find "Pre or Post" column' if pre_or_post_index.nil?
+          ProcessSpreadsheet.call(filename: filename, headers: :downcase) do |headers, row, row_index|
+            unless initialized
+              question_stem_index ||= headers.index do |header|
+                header&.start_with?('question') || header&.end_with?('stem')
+              end
+              raise ArgumentError, 'Could not find "Question Stem" column' if question_stem_index.nil?
 
-            question_stem_index ||= headers.index do |header|
-              header&.start_with?('question') || header&.end_with?('stem')
-            end
-            raise ArgumentError, 'Could not find "Question Stem" column' if question_stem_index.nil?
+              uuid_index ||= headers.index { |header| header == 'uuid' || header == 'page uuid' }
+              section_index ||= headers.index { |header| header == 'section' }
+              Rails.logger.warn { 'Could not find "UUID" or "Section" columns' } \
+                if uuid_index.nil? && section_index.nil?
 
-            answer_choice_indices ||= headers.filter_map.with_index do |header, index|
-              index if (header&.start_with?('answer') || header&.end_with?('choice')) && !header.include?('feedback')
-            end
-            raise ArgumentError, 'Could not find "Answer Choice" columns' if answer_choice_indices.empty?
+              unless section_index.nil?
+                book = OpenStax::Content::Abl.new.approved_books.find { |book| book.uuid == book_uuid }
+                book.all_pages.each { |page| page_uuid_by_book_location[page.book_location] = page.uuid }
+                raise ArgumentError, "Could not find book with UUID #{book_uuid} in the ABL" if book.nil?
+              end
 
-            feedback_indices ||= headers.filter_map.with_index do |header, index|
-              index if header&.include?('feedback')
-            end
+              nickname_index ||= headers.index { |header| header&.include?('nickname') }
 
-            correct_answer_index ||= headers.index { |header| header&.start_with?('correct') }
-            raise ArgumentError, 'Could not find "Correct Answer" column' if correct_answer_index.nil?
+              pre_or_post_index ||= headers.index { |header| header&.start_with?('pre') && header.end_with?('post') }
+              Rails.logger.warn { 'Could not find "Pre or Post" column' } if pre_or_post_index.nil?
 
-            detailed_solution_index ||= headers.index { |header| header&.end_with?('solution') }
-            raise ArgumentError, 'Could not find "Detailed Solution" column' if detailed_solution_index.nil?
+              teks_index ||= headers.index { |header| header&.include?('teks') && !header.include?('machine') }
+              machine_teks_index ||= headers.index { |header| header&.include?('machine') && header.include?('teks') }
 
-            row_number = row_index + 1
+              raise_id_index ||= headers.index { |header| header&.include?('raise') }
 
-            page_uuid = if uuid_index.nil? || row[uuid_index].blank?
-              page_uuid_by_book_location[row[section_index].split('.').map(&:to_i)] unless row[section_index].blank?
-            else
-              row[uuid_index]
-            end
+              background_index ||= headers.index { |header| header&.include?('background') }
+
+              multi_step_index ||= headers.index { |header| header&.include?('multi') && header.include?('step') }
+
+              block_index ||= headers.index { |header| header&.include?('block') }
 
-            if page_uuid.blank?
-              Rails.logger.info { "Skipped row ##{row_number} with blank Section or Page UUID" }
-              next
+              answer_choice_indices ||= headers.filter_map.with_index do |header, index|
+                index if (
+                  header&.start_with?('answer') || header&.start_with?('option') || header&.end_with?('choice')
+                ) && !header.include?('feedback')
+              end
+              Rails.logger.warn { 'Could not find "Answer Choice" columns' } if answer_choice_indices.empty?
+
+              correct_answer_index ||= headers.index { |header| header&.start_with?('correct') }
+              Rails.logger.warn { 'Could not find "Correct Answer" column' } if correct_answer_index.nil?
+
+              feedback_indices ||= headers.filter_map.with_index do |header, index|
+                index if header&.include?('feedback')
+              end
+
+              detailed_solution_index ||= headers.index { |header| header&.end_with?('solution') }
+              Rails.logger.warn { 'Could not find "Detailed Solution" column' } if detailed_solution_index.nil?
+
+              initialized = true
             end
 
-            exercise = Exercise.new
+            row_number = row_index + 1
 
-            exercise.tags = [
-              "assessment:#{row[pre_or_post_index].downcase == 'pre' ? 'preparedness' : 'practice'
-                }:https://openstax.org/orn/book:page/#{book_uuid}:#{page_uuid}",
-              "context-cnxmod:#{page_uuid}"
-            ]
+            # Using row_index here because dealing with the previous row
+            if multi_step_index.nil? || multi_step.nil? || row[multi_step_index] != multi_step
+              save.call(exercise, row_index)
+
+              multi_step = row[multi_step_index] unless multi_step_index.nil?
+
+              exercise = Exercise.new
+
+              page_uuid = if uuid_index.nil? || row[uuid_index].blank?
+                page_uuid_by_book_location[row[section_index].split('.').map(&:to_i)] \
+                  unless section_index.nil? || row[section_index].blank?
+              else
+                row[uuid_index]
+              end
+              if page_uuid.blank?
+                Rails.logger.warn { "Row ##{row_number} has no associated page in the book" } \
+                  unless uuid_index.nil? && section_index.nil?
+              else
+                exercise.tags << "context-cnxmod:#{page_uuid}"
+                exercise.tags << "assessment:#{row[pre_or_post_index].downcase == 'pre' ? 'preparedness' : 'practice'
+                  }:https://openstax.org/orn/book:page/#{book_uuid}:#{page_uuid}" \
+                  unless pre_or_post_index.nil? || row[pre_or_post_index].blank?
+              end
+
+              unless teks_index.nil? || row[teks_index].blank?
+                teks = row[teks_index].split(/,|;/).map(&:strip)
+                teks.each { |tek| exercise.tags << "teks:#{tek}" }
+              end
+              unless machine_teks_index.nil? || row[machine_teks_index].blank?
+                machine_teks = row[machine_teks_index].split(/,|;/).map(&:strip)
+                machine_teks.each { |tek| exercise.tags << "machine-teks:#{tek}" }
+              end
+              exercise.tags << "raise-content-id:#{row[raise_id_index]}" \
+                unless raise_id_index.nil? || row[raise_id_index].blank?
+
+              exercise.publication.authors << Author.new(user: author)
+              exercise.publication.copyright_holders << CopyrightHolder.new(user: copyright_holder)
+
+              exercise.publication.publication_group.nickname = row[nickname_index] unless nickname_index.nil?
+
+              exercise.stimulus = parse(row[background_index], exercise) unless background_index.nil?
+            else
+              Rails.logger.info { "Imported row ##{row_index} - Multi-step exercise" }
+            end
 
             question = Question.new
+            question.sort_position = row[block_index].to_i + 1 unless block_index.nil?
             exercise.questions << question
 
             stem = Stem.new(content: parse(row[question_stem_index], exercise))
             stem.stylings << Styling.new(style: ::Style::MULTIPLE_CHOICE)
             question.stems << stem
 
-            exercise.publication.authors << Author.new(user: author)
-            exercise.publication.copyright_holders << CopyrightHolder.new(user: copyright_holder)
+            unless detailed_solution_index.nil? || row[detailed_solution_index].blank?
+              solution = CollaboratorSolution.new(
+                solution_type: SolutionType::DETAILED,
+                content: parse(row[detailed_solution_index], exercise)
+              )
+              question.collaborator_solutions << solution
+            end
+
+            next if correct_answer_index.nil? || row[correct_answer_index].blank?
 
             correct_answer = row[correct_answer_index].downcase.strip.each_byte.first - 97
             answer_choice_indices.each_with_index do |row_index, answer_index|
@@ -113,26 +195,9 @@ def exec(filename:, book_uuid:)
                 feedback: parse(feedback, exercise)
               )
             end
-
-            detailed_solution = row[detailed_solution_index]
-            if detailed_solution.present?
-              solution = CollaboratorSolution.new(
-                solution_type: SolutionType::DETAILED,
-                content: parse(detailed_solution, exercise)
-              )
-              question.collaborator_solutions << solution
-            end
-
-            begin
-              exercise.save!
-              exercise.publication.publish.save!
-
-              Rails.logger.info { "Imported row ##{row_number} - New exercise ID: #{exercise.uid}" }
-            rescue StandardError => error
-              Rails.logger.error { "Failed to import row ##{row_number} - #{error.message}" }
-              failures[row_number] = error.to_s
-            end
           end
+
+          save.call(exercise, row_number)
         end
       end
     end

app/routines/process_spreadsheet.rb

@@ -18,8 +18,8 @@ def exec(filename:, offset: 1, pad_cells: true, headers: false, &block)
 
     args = []
     pad_to_size = 0 if pad_cells
-    klass.new(filename).public_send(method).each_with_index do |row, row_index|
-      normalized_row = row.map { |cell| cell.value&.to_s&.strip }
+    klass.new(filename).public_send(method, pad_cells: pad_cells).each_with_index do |row, row_index|
+      normalized_row = row.map { |cell| cell&.value&.to_s&.strip }
 
       if headers && row_index == 0
         header_row = normalized_row

lib/openstax_kramdown.rb

@@ -0,0 +1,64 @@
+require 'addressable/uri'
+
+class Kramdown::Parser::Openstax < Kramdown::Parser::Html
+  ESCAPED_PARENS_MATH_RE = /(.*?)\\\((.*?)\\\)(.*)/
+
+  def secrets
+    @secrets ||= Rails.application.secrets
+  end
+
+  def s3_secrets
+    @s3_secrets ||= secrets.aws[:s3]
+  end
+
+  def region
+    @region ||= s3_secrets[:region]
+  end
+
+  def s3_client
+    @s3_client ||= Aws::S3::Client.new(
+      region: region,
+      credentials: Aws::Credentials.new(s3_secrets[:access_key_id], s3_secrets[:secret_access_key])
+    )
+  end
+
+  def add_text(text, tree = @tree, type = @text_type)
+    matches = ESCAPED_PARENS_MATH_RE.match(text)
+    return super(text, tree, type) if matches.nil?
+
+    super(matches[1], tree, type)
+
+    math = matches[2].strip
+    last = tree.children.last
+    location = (last && last.options[:location] || tree.options[:location])
+    math_el = Kramdown::Element.new(:html_element, 'span', { 'data-math' => math }, location: location)
+    tree.children << math_el
+    super(math, math_el, :text)
+
+    add_text(matches[3], tree, type)
+  end
+
+  def handle_html_start_tag(line, &block)
+    super(line, &block).tap do
+      next if Rails.env.development? || !@options[:attachable]
+      last_el = @tree.children.last
+      next unless last_el.type == :html_element && last_el.value == 'img' && last_el.attr['src']
+
+      uri = URI.parse(last_el.attr['src'])
+      contents = Net::HTTP.get(uri)
+
+      bucket_name = s3_secrets[:uploads_bucket_name]
+      key = "#{secrets.environment_name}/#{Digest::SHA2.new.update(contents).to_s}#{File.extname(uri.path)}"
+
+      s3_client.put_object(
+        body: StringIO.new(contents),
+        bucket: bucket_name,
+        key: key
+      )
+
+      last_el.attr['src'] = "https://s3.#{region}.amazonaws.com/#{bucket_name}/#{key}"
+
+      @options[:attachable].attachments << Attachment.new(asset: last_el.attr['src'])
+    end
+  end
+end

lib/row_parser.rb

@@ -1,17 +1,17 @@
+require 'openstax_kramdown'
+
 # Common methods for reading spreadsheets
 module RowParser
   # Parses the text using Markdown
   # Attachments are associated with the given Exercise object
   def parse(text, exercise)
     return nil if text.blank?
 
-    text = text.to_s
-
-    kd = Kramdown::Document.new(text.to_s.strip, math_engine: :openstax, attachable: exercise)
-    # If only one <p> tag, remove it and just return the nodes below
+    kd = Kramdown::Document.new(text.to_s.strip, input: 'Openstax', attachable: exercise)
+    # If only one <p> tag after Kramdown parsing, remove it and just return the nodes below
     kd.root.children = kd.root.children.first.children \
       if kd.root.children.length == 1 && kd.root.children.first.type == :p
-    kd.to_html
+    kd.to_html.strip
   end
 
   def record_failures