Properly compute sense definition

Author: sheerun

app/assets/javascripts/directives/sense_tooltip.coffee

@@ -9,6 +9,6 @@ angular.module('wordnet').directive 'senseTooltip', ->
       if sense?
         title += sense.lemma
         title += " (#{sense.part_of_speech})" if sense.part_of_speech
-        title += " — #{sense.comment}" if sense.comment
+        title += " — #{sense.definition}" if sense.definition
 
       element.attr('title', title)

app/controllers/api/lexemes.rb

@@ -71,7 +71,7 @@ class Lexemes < Grape::API
               id: e2.id,
               lemma: e2.lemma,
               sense_index: e2.sense_index,
-              comment: e2.comment
+              definition: e2.definition
             })
           )
         """.gsub(/\s+/, ' ').strip.freeze

app/exporters/neo4j/sense.rb

@@ -6,15 +6,15 @@ class Sense < Exporter
 
       ON CREATE SET
       n.domain_id = {domain_id},
-      n.comment = {comment},
+      n.definition = {definition},
       n.sense_index = {sense_index},
       n.language = {language},
       n.part_of_speech = {part_of_speech},
       n.lemma = {lemma}
 
       ON MATCH SET
       n.domain_id = {domain_id},
-      n.comment = {comment},
+      n.definition = {definition},
       n.sense_index = {sense_index},
       n.language = {language},
       n.part_of_speech = {part_of_speech},

app/importers/importer.rb

@@ -19,19 +19,29 @@ def table_name
     raise NotImplementedError.new("You must implement table_name.")
   end
 
-  def initialize(options = {})
-    @base_connection = options.fetch(:connection, ActiveRecord::Base.connection)
-    @sequel_connection = Sequel.connect(
+  def base_connection
+    @base_connection ||=
+      options.fetch(:connection, ActiveRecord::Base.connection)
+  end
+
+  def sequel_connection
+    @sequel_connection ||= Sequel.connect(
       Rails.configuration.database_configuration[Rails.env].
         merge("adapter" => "postgres")
     )
+  end
+  
+  def pages
+    @pages ||= (total_count.to_f / @batch_size).ceil
+  end
 
+  def initialize(options = {})
     @batch_size = options.fetch(:batch_size, 400)
-    @pages = options.fetch(:pages, (total_count.to_f / @batch_size).ceil)
+    @pages = options.fetch(:pages, nil)
   end
 
   def import!
-    (0...@pages).each do |page|
+    (0...self.pages).each do |page|
       import_entities!(@batch_size, page * @batch_size)
     end
   end
@@ -51,7 +61,7 @@ def process_entities!(entities)
   def process_uuid_mappings(entities, uuid_mappings)
     uuid_mappings.each do |key, opts|
       uuid_mapping = Hash[
-        @sequel_connection[opts[:table]].select(:id, opts[:attribute]).
+        self.sequel_connection[opts[:table]].select(:id, opts[:attribute]).
         where(opts[:attribute] => entities.map { |w| w[key] }).to_a.
         map { |w| [w[opts[:attribute]], w[:id]] }
       ]
@@ -67,7 +77,7 @@ def process_uuid_mappings(entities, uuid_mappings)
   end
 
   def persist_entities!(table_name, entities, unique_attributes)
-    Upsert.batch(@base_connection, table_name) do |upsert|
+    Upsert.batch(self.base_connection, table_name) do |upsert|
       entities.each do |hash|
         unique_map = hash.extract!(*unique_attributes)
         upsert.row(unique_map, hash)

app/importers/progress_bar_importer.rb

@@ -2,7 +2,7 @@ module ProgressBarImporter
   def import!(*args, &block)
     @progress_bar = ProgressBar.create(
       :title => self.class.name.split('::').last,
-      :total => @pages,
+      :total => self.pages,
       :format => "%t: |%B| %c/%C %E",
       :smoothing => 0.8
     )

app/importers/wordnet_pl/sense.rb

@@ -51,7 +51,6 @@ def load_entities(limit, offset)
           external_id: lemma[:ID],
           domain_id: lemma[:domain],
           lemma: lemma[:lemma],
-          comment: extract_short_definition(lemma[:comment]),
           examples: extract_examples(lemma[:comment]),
           definition: extract_definition(lemma[:comment]),
           language: lemma[:project] > 0 ? 'pl_PL' : 'en_GB',
@@ -67,14 +66,6 @@ def convert_pos(pos_id)
       PartOfSpeech.find(pos_id).uuid
     end
 
-    def process_comment(comment)
-      return nil if comment.blank?
-      return nil if comment.include?("brak danych")
-      return nil if comment.include?("##")
-      return nil if comment[0..1] == "NP"
-      comment.match(/^([^#<]+)/).to_s.presence
-    end
-
     def extract_examples(comment)
       comment.
         scan(/\[##[^:]+: ([^\]]*)\]/).
@@ -83,14 +74,14 @@ def extract_examples(comment)
         reject(&:empty?)
     end
 
-    def extract_short_definition(comment)
-      definition = comment.scan(/##D: ([^\.]+)\./).flatten.first
-      definition && definition.size < 128 ? definition : nil
-    end
-
     def extract_definition(comment)
-      definition = comment.scan(/##D: ([^\.]+)\./).flatten.first
-      definition && definition.size >= 128 ? definition : nil
+      comment.
+        scan(/##D:\s*(.+?)\.\s*(?:\z|\[##|\{##L:|<#)/m).
+        flatten.
+        first.
+        to_s.
+        strip.
+        presence
     end
   end
 end

app/importers/wordnet_pl/synset.rb

@@ -15,7 +15,7 @@ def total_count
     end
 
     def load_entities(limit, offset)
-      raw = @connection[:synset].select(:ID, :comment, :definition).order(:ID).
+      raw = @connection[:synset].select(:ID, :definition).order(:ID).
         where('ID >= ? AND ID < ?', offset, offset + limit).to_a
 
       raw.map do |synset|

app/models/sense.rb

@@ -46,7 +46,7 @@ def fetch_related
           id: target.id,
           lemma: target.lemma,
           domain_id: target.domain_id,
-          comment: target.comment,
+          definition: target.definition,
           part_of_speech: target.part_of_speech,
           sense_index: target.sense_index
         })
@@ -73,7 +73,7 @@ def fetch_reverse_related
           id: target.id,
           lemma: target.lemma,
           domain_id: target.domain_id,
-          comment: target.comment,
+          definition: target.definition,
           part_of_speech: target.part_of_speech,
           sense_index: target.sense_index
         })
@@ -93,7 +93,7 @@ def as_json(options = {})
       :language => language,
       :domain_id => domain_id,
       :part_of_speech => part_of_speech,
-      :comment => comment
+      :definition => definition
     }
 
     if options[:extended]
@@ -106,7 +106,6 @@ def as_json(options = {})
       ).as_json
 
       data[:synset] = synset.as_json(without: id)
-      data[:definition] = definition
       data[:examples] = examples || synset.examples || []
       data[:outgoing] = fetch_related
       data[:incoming] = fetch_reverse_related

app/models/synset.rb

@@ -17,7 +17,6 @@ def as_json(options = {})
 
     {
       senses: the_senses.as_json,
-      comment: comment,
       definition: definition
     }
   end

app/views/home/index.slim

@@ -14,11 +14,11 @@
                   span.u__bold> ng-bind-html="homograph.lemma"
                   small> ng-bind="homograph.sense_index"
                   small.u__dim<(
-                    ng-if="homograph.comment"
-                    ng-bind-html="homograph.comment | highlight: $select.search"
+                    ng-if="homograph.definition"
+                    ng-bind-html="homograph.definition | highlight: $select.search"
                   )
                   small.u__dim<(
-                    ng-if="!homograph.comment"
+                    ng-if="!homograph.definition"
                     ng-bind-html="('domain_' + homograph.domain_id) | translate | highlight: $select.search"
                   )
                 div style="position: absolute; right: 0px; top: -1px"