Add the :jsonify_keyset_attributes variable to override the encoding (#749)

Author: ddnexus

.rubocop.yml

@@ -22,6 +22,9 @@ Layout/LineLength:
   Exclude:
   - test/**/*
 
+Layout/BeginEndAlignment:
+  EnforcedStyleAlignWith: begin
+
 Layout/ExtraSpacing:
   AllowForAlignment:
     Enabled: true

docs/api/keyset.md

@@ -114,7 +114,7 @@ If you need a specific order:
 ### How Pagy::Keyset works
 
 - You pass an `uniquely ordered` `set` and `Pagy::Keyset` queries the page of records.
-- It keeps track of the `latest` fetched  record by encoding its `keyset` attributes into the `page` query string param of the 
+- It keeps track of the `latest` fetched record by encoding its `keyset` attributes into the `page` query string param of the 
   `next` URL.
 - At each request, the `:page` is decoded and used to prepare a `when` clause that filters the newest records, and 
   the `:limit` of records is pulled.
@@ -196,6 +196,37 @@ end
 Pagy::Keyset(set, typecast_latest:)
 ```
 
+==- `:jsonify_keyset_attributes`
+
+A lambda to override the generic json encoding of the `keyset` attributes. Use it when the generic `to_json` method would lose 
+some information when decoded.
+
+For example: `Time` objects may lose or round the fractional seconds through the 
+encoding/decoding cycle, causing the ordering to fail and thus creating all sort of unexpected behaviors (e.g. skipping or 
+repeating the same page, missing or duplicated records, etc.). Here is what you can do:
+
+```ruby
+# Match the microsecods with the strings stored into the time columns of SQLite
+jsonify_keyset_attributes = lambda do |attributes|
+  # Convert it to a string matching the stored value/format in SQLite DB
+  attributes[:created_at] = attributes[:created_at].strftime('%F %T.%6N')
+  attributes.to_json
+end
+
+Pagy::Keyset(set, jsonify_keyset_attributes:)
+```
+
+!!! ActiveRecord alternative for time_precision
+
+With `ActiveRecord::Relation` set, you can fix the fractional seconds issue by just setting the `time_precision`: 
+
+```ruby
+ActiveSupport::JSON::Encoding.time_precision = 6
+```
+!!!
+
+_(Notice that it doesn't work with `Sequel::Dataset` sets)_
+
 ===
 
 ## Attribute Readers
@@ -220,8 +251,17 @@ Product.order(:name, :production_date)
 Product.order(:name, :production_date, :id)
 ```
 !!!
+ 
+!!!danger You may have an encoding problem
+The generic `to_json` method used to encode the `page` loses some information when decoded.
+
+!!!success
+- Check the actual executed DB query and the actual stored value
+- Identify the column that have a format that doesn't match with the keyset
+- Use your custom encoding with the [:jsonify_keyset_attributes](#jsonify-keyset-attributes) variable
+!!!
 
-!!!danger ... or you have a typecasting problem
+!!!danger You may have a typecasting problem
 Your ORM and the storage formats don't match for one or more columns. It's a common case with `SQLite` and Time columns.
 They may have been stored as strings formatted differently than the default format used by your current ORM.
 

gem/apps/keyset_ar.ru

@@ -138,6 +138,10 @@ end
 
 # ActiveRecord setup
 require 'active_record'
+
+# Match the microsecods with the strings stored into the time columns of SQLite
+# ActiveSupport::JSON::Encoding.time_precision = 6
+
 # Log
 output = ENV['APP_ENV'].equal?('showcase') ? IO::NULL : $stdout
 ActiveRecord::Base.logger = Logger.new(output)

gem/lib/pagy/keyset.rb

@@ -53,24 +53,28 @@ def next
       records
       return unless @more
 
-      @next ||= B64.urlsafe_encode(latest_from(@records.last).to_json)
+      @next ||= begin
+                  hash = keyset_attributes_from(@records.last)
+                  json = @vars[:jsonify_keyset_attributes]&.(hash) || hash.to_json
+                  B64.urlsafe_encode(json)
+                end
     end
 
     # Fetch the array of records for the current page
     def records
       @records ||= begin
-        @set = apply_select if select?
-        if @latest
-          # :nocov:
-          @set = @vars[:after_latest]&.(@set, @latest) || # deprecated
-                 # :nocov:
-                 @vars[:filter_newest]&.(@set, @latest, @keyset) ||
-                 filter_newest
-        end
-        records = @set.limit(@limit + 1).to_a
-        @more   = records.size > @limit && !records.pop.nil?
-        records
-      end
+                     @set = apply_select if select?
+                     if @latest
+                       # :nocov:
+                       @set = @vars[:after_latest]&.(@set, @latest) || # deprecated
+                              # :nocov:
+                              @vars[:filter_newest]&.(@set, @latest, @keyset) ||
+                              filter_newest
+                     end
+                     records = @set.limit(@limit + 1).to_a
+                     @more   = records.size > @limit && !records.pop.nil?
+                     records
+                   end
     end
 
     protected

gem/lib/pagy/keyset/active_record.rb

@@ -7,8 +7,8 @@ class Keyset
     class ActiveRecord < Keyset
       protected
 
-      # Get the keyset attributes of the record
-      def latest_from(latest_record) = latest_record.slice(*@keyset.keys)
+      # Get the keyset attributes from the record
+      def keyset_attributes_from(record) = record.slice(*@keyset.keys)
 
       # Extract the keyset from the set
       def extract_keyset

gem/lib/pagy/keyset/sequel.rb

@@ -7,8 +7,8 @@ class Keyset
     class Sequel < Keyset
       protected
 
-      # Get the keyset attributes of the latest record
-      def latest_from(latest_record) = latest_record.to_hash.slice(*@keyset.keys)
+      # Get the keyset attributes from the record
+      def keyset_attributes_from(record) = record.to_hash.slice(*@keyset.keys)
 
       # Extract the keyset from the set
       def extract_keyset

test/pagy/keyset_test.rb

@@ -49,6 +49,14 @@
         _ = pagy.records
         _(pagy.latest).must_equal({id: 10})
       end
+      it 'uses :jsonify_keyset_attributes' do
+        pagy = Pagy::Keyset.new(model.order(:id),
+                                page: "eyJpZCI6MTB9",
+                                limit: 10,
+                                jsonify_keyset_attributes: lambda(&:to_json))
+        _(pagy.next).must_equal("eyJpZCI6MjB9")
+        _(pagy.latest).must_equal({id: 10})
+      end
       it 'uses :filter_newest' do
         filter_newest = if model == Pet
                           ->(set, latest, _keyset) { set.where('id > :id', **latest) }