Cpfm 246

.rspec

@@ -1 +1 @@
---require spec_helper --tag ~kafka_integration --tag ~integration --format documentation
+--require spec_helper --tag ~kafka_integration --tag integration --format documentation

CHANGELOG.md

@@ -6,6 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## UNRELEASED
+- DB Poller orders results by record class timestamps, to prevent ambiguity from joins
 
 ## 1.11.0 - 2021-08-27
 

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    deimos-ruby (1.10.2)
+    deimos-ruby (1.11.0)
       avro_turf (~> 0.11)
       fig_tree (~> 0.0.2)
       phobos (>= 1.9, < 3.0)
@@ -11,62 +11,62 @@ PATH
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (6.1.3)
-      actionpack (= 6.1.3)
-      activesupport (= 6.1.3)
+    actioncable (6.1.4.1)
+      actionpack (= 6.1.4.1)
+      activesupport (= 6.1.4.1)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailbox (6.1.3)
-      actionpack (= 6.1.3)
-      activejob (= 6.1.3)
-      activerecord (= 6.1.3)
-      activestorage (= 6.1.3)
-      activesupport (= 6.1.3)
+    actionmailbox (6.1.4.1)
+      actionpack (= 6.1.4.1)
+      activejob (= 6.1.4.1)
+      activerecord (= 6.1.4.1)
+      activestorage (= 6.1.4.1)
+      activesupport (= 6.1.4.1)
       mail (>= 2.7.1)
-    actionmailer (6.1.3)
-      actionpack (= 6.1.3)
-      actionview (= 6.1.3)
-      activejob (= 6.1.3)
-      activesupport (= 6.1.3)
+    actionmailer (6.1.4.1)
+      actionpack (= 6.1.4.1)
+      actionview (= 6.1.4.1)
+      activejob (= 6.1.4.1)
+      activesupport (= 6.1.4.1)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (6.1.3)
-      actionview (= 6.1.3)
-      activesupport (= 6.1.3)
+    actionpack (6.1.4.1)
+      actionview (= 6.1.4.1)
+      activesupport (= 6.1.4.1)
       rack (~> 2.0, >= 2.0.9)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (6.1.3)
-      actionpack (= 6.1.3)
-      activerecord (= 6.1.3)
-      activestorage (= 6.1.3)
-      activesupport (= 6.1.3)
+    actiontext (6.1.4.1)
+      actionpack (= 6.1.4.1)
+      activerecord (= 6.1.4.1)
+      activestorage (= 6.1.4.1)
+      activesupport (= 6.1.4.1)
       nokogiri (>= 1.8.5)
-    actionview (6.1.3)
-      activesupport (= 6.1.3)
+    actionview (6.1.4.1)
+      activesupport (= 6.1.4.1)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activejob (6.1.3)
-      activesupport (= 6.1.3)
+    activejob (6.1.4.1)
+      activesupport (= 6.1.4.1)
       globalid (>= 0.3.6)
-    activemodel (6.1.3)
-      activesupport (= 6.1.3)
-    activerecord (6.1.3)
-      activemodel (= 6.1.3)
-      activesupport (= 6.1.3)
-    activerecord-import (1.0.8)
+    activemodel (6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activerecord (6.1.4.1)
+      activemodel (= 6.1.4.1)
+      activesupport (= 6.1.4.1)
+    activerecord-import (1.2.0)
       activerecord (>= 3.2)
-    activestorage (6.1.3)
-      actionpack (= 6.1.3)
-      activejob (= 6.1.3)
-      activerecord (= 6.1.3)
-      activesupport (= 6.1.3)
-      marcel (~> 0.3.1)
-      mimemagic (~> 0.3.2)
-    activesupport (6.1.3)
+    activestorage (6.1.4.1)
+      actionpack (= 6.1.4.1)
+      activejob (= 6.1.4.1)
+      activerecord (= 6.1.4.1)
+      activesupport (= 6.1.4.1)
+      marcel (~> 1.0.0)
+      mini_mime (>= 1.1.0)
+    activesupport (6.1.4.1)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
@@ -80,126 +80,123 @@ GEM
       excon (~> 0.45)
     builder (3.2.4)
     coderay (1.1.3)
-    concurrent-ruby (1.1.8)
-    concurrent-ruby-ext (1.1.8)
-      concurrent-ruby (= 1.1.8)
+    concurrent-ruby (1.1.9)
+    concurrent-ruby-ext (1.1.9)
+      concurrent-ruby (= 1.1.9)
     crass (1.0.6)
     database_cleaner (1.99.0)
-    ddtrace (0.46.0)
+    ddtrace (0.52.0)
+      ffi (~> 1.0)
       msgpack
     diff-lcs (1.4.4)
-    digest-crc (0.6.3)
+    digest-crc (0.6.4)
       rake (>= 12.0.0, < 14.0.0)
     dogstatsd-ruby (4.8.3)
     erubi (1.10.0)
-    excon (0.81.0)
+    excon (0.86.0)
     exponential-backoff (0.0.4)
-    ffi (1.15.0)
+    ffi (1.15.4)
     fig_tree (0.0.2)
       activesupport (>= 3.0.0)
-    formatador (0.2.5)
-    globalid (0.4.2)
-      activesupport (>= 4.2.0)
-    guard (2.16.2)
+    formatador (0.3.0)
+    globalid (0.5.2)
+      activesupport (>= 5.0)
+    guard (2.18.0)
       formatador (>= 0.2.4)
       listen (>= 2.7, < 4.0)
       lumberjack (>= 1.0.12, < 2.0)
       nenv (~> 0.1)
       notiffany (~> 0.0)
-      pry (>= 0.9.12)
+      pry (>= 0.13.0)
       shellany (~> 0.0)
       thor (>= 0.18.1)
     guard-compat (1.2.1)
     guard-rspec (4.7.3)
       guard (~> 2.1)
       guard-compat (~> 1.1)
       rspec (>= 2.99.0, < 4.0)
-    guard-rubocop (1.4.0)
+    guard-rubocop (1.5.0)
       guard (~> 2.0)
       rubocop (< 2.0)
-    i18n (1.8.9)
+    i18n (1.8.10)
       concurrent-ruby (~> 1.0)
-    listen (3.4.1)
+    listen (3.7.0)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
     little-plugger (1.1.4)
     logging (2.3.0)
       little-plugger (~> 1.1)
       multi_json (~> 1.14)
-    loofah (2.9.0)
+    loofah (2.12.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     lumberjack (1.2.8)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
-    marcel (0.3.3)
-      mimemagic (~> 0.3.2)
+    marcel (1.0.2)
     method_source (1.0.0)
-    mimemagic (0.3.10)
-      nokogiri (~> 1)
-      rake
-    mini_mime (1.0.2)
-    mini_portile2 (2.5.1)
+    mini_mime (1.1.1)
+    mini_portile2 (2.6.1)
     minitest (5.14.4)
     msgpack (1.4.2)
     multi_json (1.15.0)
     mysql2 (0.5.3)
     nenv (0.3.0)
-    nio4r (2.5.7)
-    nokogiri (1.11.5)
-      mini_portile2 (~> 2.5.0)
+    nio4r (2.5.8)
+    nokogiri (1.12.5)
+      mini_portile2 (~> 2.6.1)
       racc (~> 1.4)
     notiffany (0.1.3)
       nenv (~> 0.1)
       shellany (~> 0.0)
-    parallel (1.20.1)
-    parser (3.0.0.0)
+    parallel (1.21.0)
+    parser (3.0.2.0)
       ast (~> 2.4.1)
     pg (1.2.3)
-    phobos (2.0.2)
+    phobos (2.1.0)
       activesupport (>= 3.0.0)
       concurrent-ruby (>= 1.0.2)
       concurrent-ruby-ext (>= 1.0.2)
       exponential-backoff
       logging
       ruby-kafka
       thor
-    pry (0.14.0)
+    pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
     racc (1.5.2)
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (6.1.3)
-      actioncable (= 6.1.3)
-      actionmailbox (= 6.1.3)
-      actionmailer (= 6.1.3)
-      actionpack (= 6.1.3)
-      actiontext (= 6.1.3)
-      actionview (= 6.1.3)
-      activejob (= 6.1.3)
-      activemodel (= 6.1.3)
-      activerecord (= 6.1.3)
-      activestorage (= 6.1.3)
-      activesupport (= 6.1.3)
+    rails (6.1.4.1)
+      actioncable (= 6.1.4.1)
+      actionmailbox (= 6.1.4.1)
+      actionmailer (= 6.1.4.1)
+      actionpack (= 6.1.4.1)
+      actiontext (= 6.1.4.1)
+      actionview (= 6.1.4.1)
+      activejob (= 6.1.4.1)
+      activemodel (= 6.1.4.1)
+      activerecord (= 6.1.4.1)
+      activestorage (= 6.1.4.1)
+      activesupport (= 6.1.4.1)
       bundler (>= 1.15.0)
-      railties (= 6.1.3)
+      railties (= 6.1.4.1)
       sprockets-rails (>= 2.0.0)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.3.0)
+    rails-html-sanitizer (1.4.2)
       loofah (~> 2.3)
-    railties (6.1.3)
-      actionpack (= 6.1.3)
-      activesupport (= 6.1.3)
+    railties (6.1.4.1)
+      actionpack (= 6.1.4.1)
+      activesupport (= 6.1.4.1)
       method_source
-      rake (>= 0.8.7)
+      rake (>= 0.13)
       thor (~> 1.0)
     rainbow (3.0.0)
-    rake (13.0.3)
-    rb-fsevent (0.10.4)
+    rake (13.0.6)
+    rb-fsevent (0.11.0)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
     regexp_parser (2.1.1)
@@ -216,7 +213,7 @@ GEM
     rspec-mocks (3.10.2)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.10.0)
-    rspec-rails (4.1.1)
+    rspec-rails (4.1.2)
       actionpack (>= 4.2)
       activesupport (>= 4.2)
       railties (>= 4.2)
@@ -240,7 +237,7 @@ GEM
       parser (>= 2.7.1.5)
     rubocop-rspec (1.42.0)
       rubocop (>= 0.87.0)
-    ruby-kafka (1.3.0)
+    ruby-kafka (1.4.0)
       digest-crc
     ruby-progressbar (1.11.0)
     shellany (0.0.1)
@@ -258,8 +255,8 @@ GEM
     thor (1.1.0)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (1.7.0)
-    websocket-driver (0.7.3)
+    unicode-display_width (1.8.0)
+    websocket-driver (0.7.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     zeitwerk (2.4.2)

lib/deimos/utils/db_poller.rb

@@ -127,12 +127,13 @@ def fetch_results(time_from, time_to)
         id = @producer.config[:record_class].primary_key
         quoted_timestamp = ActiveRecord::Base.connection.quote_column_name(@config.timestamp_column)
         quoted_id = ActiveRecord::Base.connection.quote_column_name(id)
+        table = @producer.config[:record_class].table_name
         @producer.poll_query(time_from: time_from,
                              time_to: time_to,
                              column_name: @config.timestamp_column,
                              min_id: @info.last_sent_id).
           limit(BATCH_SIZE).
-          order("#{quoted_timestamp}, #{quoted_id}")
+          order("#{table}.#{quoted_timestamp}, #{table}.#{quoted_id}")
       end
 
       # @param batch [Array<ActiveRecord::Base>]

spec/schemas/com/my-namespace/MySchemaJointWidgets.avsc

@@ -0,0 +1,23 @@
+{
+  "namespace": "com.my-namespace",
+  "name": "MySchemaJointWidgets",
+  "type": "record",
+  "doc": "Test schema",
+  "fields": [
+    {
+      "name": "test_id",
+      "type": "string",
+      "doc": "test string"
+    },
+    {
+      "name": "some_int",
+      "type": "int",
+      "doc": "test int"
+    },
+    {
+      "name": "some_other_int",
+      "type": "int",
+      "doc": "test int"
+    }
+  ]
+}

spec/spec_helper.rb

@@ -215,8 +215,20 @@ def setup_db(options)
       t.timestamps
     end
 
+    ActiveRecord::Base.connection.create_table(:joint_widgets, force: true) do |t|
+      t.integer(:widget_id)
+      t.integer(:some_other_int)
+      t.timestamps
+    end
+
+    class JointWidget < ActiveRecord::Base
+      belongs_to :widget
+    end
+
     # :nodoc:
     class Widget < ActiveRecord::Base
+      has_one :joint_widget
+
       # @return [String]
       def generated_id
         'generated_id'