diff --git a/docs/migrations.md b/docs/migrations.md index a039772..aa1186a 100644 --- a/docs/migrations.md +++ b/docs/migrations.md @@ -10,7 +10,7 @@ hypertable_options = { time_column: 'created_at', chunk_time_interval: '1 min', compress_segmentby: 'identifier', - compression_interval: '7 days' + compress_after: '7 days' } create_table(:events, id: false, hypertable: hypertable_options) do |t| @@ -43,7 +43,7 @@ hypertable_options = { chunk_time_interval: '1 min', compress_segmentby: 'symbol', compress_orderby: 'created_at', - compression_interval: '7 days' + compress_after: '7 days' } create_table :ticks, hypertable: hypertable_options, id: false do |t| t.string :symbol diff --git a/docs/toolkit_candlestick.md b/docs/toolkit_candlestick.md index 293f534..eebb68d 100644 --- a/docs/toolkit_candlestick.md +++ b/docs/toolkit_candlestick.md @@ -42,7 +42,7 @@ db do chunk_time_interval: "1 day", compress_segmentby: "symbol", compress_orderby: "time", - compression_interval: "1 month" + compress_after: "1 month" } create_table :ticks, hypertable: hypertable_options, id: false do |t| t.timestamp :time diff --git a/docs/toolkit_ohlc.md b/docs/toolkit_ohlc.md index 2e317a9..30100dc 100644 --- a/docs/toolkit_ohlc.md +++ b/docs/toolkit_ohlc.md @@ -22,7 +22,7 @@ hypertable_options = { chunk_time_interval: '1 week', compress_segmentby: 'symbol', compress_orderby: 'time', - compression_interval: '1 month' + compress_after: '1 month' } create_table :ticks, hypertable: hypertable_options, id: false do |t| t.timestampt :time diff --git a/examples/ranking/config/initializers/timescale.rb b/examples/ranking/config/initializers/timescale.rb index 84ed4bd..dcfdfc3 100644 --- a/examples/ranking/config/initializers/timescale.rb +++ b/examples/ranking/config/initializers/timescale.rb @@ -1,2 +1,5 @@ require 'timescaledb' require 'scenic' + +ActiveSupport.on_load(:active_record) { extend Timescaledb::ActsAsHypertable } + diff --git a/lib/timescaledb/migration_helpers.rb b/lib/timescaledb/migration_helpers.rb index 8c5f09e..8fa2f66 100644 --- a/lib/timescaledb/migration_helpers.rb +++ b/lib/timescaledb/migration_helpers.rb @@ -16,7 +16,7 @@ module MigrationHelpers # chunk_time_interval: '1 min', # compress_segmentby: 'identifier', # compress_orderby: 'created_at', - # compression_interval: '7 days' + # compress_after: '7 days' # } # # create_table(:events, id: false, hypertable: options) do |t| @@ -38,41 +38,31 @@ def valid_table_definition_options # :nodoc: # @see create_table with the hypertable options. def create_hypertable(table_name, time_column: 'created_at', + by_range: :created_at, chunk_time_interval: '1 week', compress_segmentby: nil, compress_orderby: 'created_at', - compression_interval: nil, - partition_column: nil, - number_partitions: nil, + compress_after: nil, + drop_after: nil, **hypertable_options) original_logger = ActiveRecord::Base.logger ActiveRecord::Base.logger = Logger.new(STDOUT) - options = ["chunk_time_interval => #{chunk_time_interval_clause(chunk_time_interval)}"] - options += hypertable_options.map { |k, v| "#{k} => #{quote(v)}" } - arguments = [ quote(table_name), - quote(time_column), - (quote(partition_column) if partition_column), - (number_partitions if partition_column), - *options + "by_range(#{quote(time_column)}, #{parse_interval(chunk_time_interval)})", + *hypertable_options.map { |k, v| "#{k} => #{quote(v)}" } ] execute "SELECT create_hypertable(#{arguments.compact.join(', ')})" - if compress_segmentby - execute <<~SQL - ALTER TABLE #{table_name} SET ( - timescaledb.compress, - timescaledb.compress_orderby = '#{compress_orderby}', - timescaledb.compress_segmentby = '#{compress_segmentby}' - ) - SQL + if compress_segmentby || compress_after + add_compression_policy(table_name, orderby: compress_orderby, segmentby: compress_segmentby, compress_after: compress_after) end - if compression_interval - execute "SELECT add_compression_policy('#{table_name}', INTERVAL '#{compression_interval}')" + + if drop_after + add_retention_policy(table_name, drop_after: drop_after) end ensure ActiveRecord::Base.logger = original_logger if original_logger @@ -146,14 +136,40 @@ def remove_continuous_aggregate_policy(table_name) execute "SELECT remove_continuous_aggregate_policy('#{table_name}')" end - def create_retention_policy(table_name, interval:) - execute "SELECT add_retention_policy('#{table_name}', INTERVAL '#{interval}')" + def create_retention_policy(table_name, drop_after:) + execute "SELECT add_retention_policy('#{table_name}', drop_after => #{parse_interval(drop_after)})" end + alias_method :add_retention_policy, :create_retention_policy + def remove_retention_policy(table_name) execute "SELECT remove_retention_policy('#{table_name}')" end + + # Enable compression policy. + # + # @param table_name [String] The name of the table. + # @param orderby [String] The column to order by. + # @param segmentby [String] The column to segment by. + # @param compress_after [String] The interval to compress after. + # @param compression_chunk_time_interval [String] In case to merge chunks. + # + # @see https://docs.timescale.com/api/latest/compression/add_compression_policy/ + def add_compression_policy(table_name, orderby:, segmentby:, compress_after: nil, compression_chunk_time_interval: nil) + options = [] + options << 'timescaledb.compress' + options << "timescaledb.compress_orderby = '#{orderby}'" if orderby + options << "timescaledb.compress_segmentby = '#{segmentby}'" if segmentby + options << "timescaledb.compression_chunk_time_interval = INTERVAL '#{compression_chunk_time_interval}'" if compression_chunk_time_interval + execute <<~SQL + ALTER TABLE #{table_name} SET ( + #{options.join(',')} + ) + SQL + execute "SELECT add_compression_policy('#{table_name}', compress_after => INTERVAL '#{compress_after}')" if compress_after + end + private # Build a string for the WITH clause of the CREATE MATERIALIZED VIEW statement. @@ -166,11 +182,11 @@ def build_with_clause_option_string(option_key, options) ",timescaledb.#{option_key}=#{value}" end - def chunk_time_interval_clause(chunk_time_interval) - if chunk_time_interval.is_a?(Numeric) - chunk_time_interval + def parse_interval(interval) + if interval.is_a?(Numeric) + interval else - "INTERVAL '#{chunk_time_interval}'" + "INTERVAL '#{interval}'" end end end diff --git a/spec/support/active_record/models.rb b/spec/support/active_record/models.rb index 60f5b45..f15f68b 100644 --- a/spec/support/active_record/models.rb +++ b/spec/support/active_record/models.rb @@ -1,4 +1,5 @@ -ActiveRecord::Base.extend Timescaledb::ActsAsHypertable +ActiveSupport.on_load(:active_record) { extend Timescaledb::ActsAsHypertable } + class Event < ActiveRecord::Base self.primary_key = "identifier" diff --git a/spec/support/active_record/schema.rb b/spec/support/active_record/schema.rb index 67eea6b..3a74e33 100644 --- a/spec/support/active_record/schema.rb +++ b/spec/support/active_record/schema.rb @@ -1,43 +1,28 @@ -def create_hypertable(table_name:, time_column_name: :created_at, options: {}) - create_table(table_name.to_sym, id: false, hypertable: options) do |t| - t.string :identifier, null: false - t.jsonb :payload - t.timestamp time_column_name.to_sym - end -end def setup_tables ActiveRecord::Schema.define(version: 1) do - create_hypertable( - table_name: :events, - time_column_name: :created_at, - options: { - time_column: 'created_at', - chunk_time_interval: '1 min', - compress_segmentby: 'identifier', - compression_interval: '7 days' - } - ) - - create_hypertable(table_name: :hypertable_with_no_options) + hypertable_options = { chunk_time_interval: '1 min', compress_segmentby: 'identifier', compress_after: '7 days' } - create_hypertable( - table_name: :hypertable_with_options, - time_column_name: :timestamp, - options: { - time_column: 'timestamp', - chunk_time_interval: '1 min', - compress_segmentby: 'identifier', - compress_orderby: 'timestamp', - compression_interval: '7 days' - } - ) + create_table(:events, id: false, hypertable: hypertable_options) do |t| + t.string :identifier, null: false + t.jsonb :payload + t.timestamptz :created_at + end - create_hypertable( - table_name: :hypertable_with_custom_time_column, - time_column_name: :timestamp, - options: { time_column: 'timestamp' } - ) + create_table(:hypertable_with_options, id: false, hypertable: { + time_column: :ts, + chunk_time_interval: '5 min', + compress_segmentby: 'identifier', + compress_orderby: 'ts', + compress_after: '15 min', + drop_after: '1 hour', + if_not_exists: true + }) do |t| + t.serial :id, primary_key: false + t.timestamptz :ts + t.string :identifier + t.index [:id, :ts], name: "index_hypertable_with_options_on_id_and_ts" + end create_table(:hypertable_with_id_partitioning, hypertable: { time_column: 'id', diff --git a/spec/timescaledb/acts_as_hypertable_spec.rb b/spec/timescaledb/acts_as_hypertable_spec.rb index df3f737..a13067f 100644 --- a/spec/timescaledb/acts_as_hypertable_spec.rb +++ b/spec/timescaledb/acts_as_hypertable_spec.rb @@ -94,36 +94,35 @@ Event.create!( identifier: "last_month", payload: {name: "bar", value: 2}, - created_at: Date.today.last_month + created_at: 1.month.ago ) } let(:event_one_day_outside_window) { Event.create!( identifier: "one_day_outside_window", payload: {name: "bax", value: 2}, - created_at: Date.today.last_month.beginning_of_month - 1.day + created_at: 1.month.ago.beginning_of_month - 1.day ) } let(:event_at_edge_of_window) { Event.create!( identifier: "at_edge_of_window", payload: {name: "bax", value: 2}, - created_at: Date.today.last_month.end_of_month + created_at: 1.month.ago.end_of_month ) } let(:event_this_month) { Event.create!( identifier: "this_month", payload: {name: "bax", value: 2}, - created_at: Date.today + created_at: Time.now ) } - it "returns all the records that were created in the previous month" do aggregate_failures do - expect(Event.previous_month).to match_array([event_last_month, event_at_edge_of_window]) + expect(Event.previous_month).to match_array([event_last_month]) expect(Event.previous_month) - .not_to include(event_one_day_outside_window, event_this_month) + .not_to include(event_one_day_outside_window, event_this_month, event_at_edge_of_window) end end end @@ -141,36 +140,36 @@ Event.create!( identifier: "last_week", payload: {name: "bar", value: 2}, - created_at: Date.today.last_week + created_at: 1.week.ago ) } let(:event_one_day_outside_window) { Event.create!( identifier: "one_day_outside_window", payload: {name: "bax", value: 2}, - created_at: Date.today.last_week.beginning_of_week - 1.day + created_at: 1.week.ago.beginning_of_week - 1.day ) } let(:event_at_edge_of_window) { Event.create!( identifier: "at_edge_of_window", payload: {name: "bax", value: 2}, - created_at: Date.today.last_week.end_of_week + created_at: 1.week.ago.end_of_week ) } let(:event_this_week) { Event.create!( identifier: "this_week", payload: {name: "bax", value: 2}, - created_at: Date.today + created_at: Time.now ) } it "returns all the records that were created in the previous week" do aggregate_failures do - expect(Event.previous_week).to match_array([event_last_week, event_at_edge_of_window]) + expect(Event.previous_week).to match_array([event_last_week]) expect(Event.previous_week) - .not_to include(event_one_day_outside_window, event_this_week) + .not_to include(event_one_day_outside_window, event_this_week, event_at_edge_of_window) end end end @@ -188,43 +187,43 @@ Event.create!( identifier: "this_month", payload: {name: "bar", value: 2}, - created_at: Date.today.beginning_of_month + created_at: Time.now.beginning_of_month ) } let(:event_one_day_outside_window) { Event.create!( identifier: "one_day_outside_window", payload: {name: "bax", value: 2}, - created_at: Date.today.beginning_of_month - 1.day + created_at: Time.now.beginning_of_month - 1.day ) } let(:event_at_edge_of_window) { Event.create!( identifier: "at_edge_of_window", payload: {name: "bax", value: 2}, - created_at: Date.today.end_of_month + created_at: Time.now.end_of_month ) } let(:event_last_month) { Event.create!( identifier: "last_month", payload: {name: "bax", value: 2}, - created_at: Date.today.last_month + created_at: 1.month.ago ) } let(:event_next_month) { Event.create!( - identifier: "next_week", + identifier: "next_month", payload: {name: "bax", value: 2}, - created_at: Date.today.next_month + created_at: 1.month.from_now ) } it "returns all the records that were created this month" do aggregate_failures do - expect(Event.this_month).to match_array([event_this_month, event_at_edge_of_window]) + expect(Event.this_month).to match_array([event_this_month]) expect(Event.this_month) - .not_to include(event_one_day_outside_window, event_last_month, event_next_month) + .not_to include(event_one_day_outside_window, event_last_month, event_next_month, event_at_edge_of_window) end end end @@ -242,35 +241,35 @@ Event.create!( identifier: "this_week", payload: {name: "bar", value: 2}, - created_at: Date.today + created_at: Time.now ) } let(:event_one_day_outside_window) { Event.create!( identifier: "one_day_outside_window", payload: {name: "bax", value: 2}, - created_at: Date.today.beginning_of_week - 1.day + created_at: 1.week.ago.beginning_of_week - 1.day ) } let(:event_at_edge_of_window) { Event.create!( identifier: "at_edge_of_window", payload: {name: "bax", value: 2}, - created_at: Date.today.end_of_week + created_at: 1.week.ago.end_of_week - 1.minute ) } let(:event_last_week) { Event.create!( identifier: "last_week", payload: {name: "bax", value: 2}, - created_at: Date.today.last_week + created_at: 1.week.ago ) } let(:event_next_week) { Event.create!( identifier: "next_week", payload: {name: "bax", value: 2}, - created_at: Date.today.next_week + created_at: 1.week.from_now ) } @@ -296,28 +295,28 @@ Event.create!( identifier: "yesterday", payload: {name: "bar", value: 2}, - created_at: Date.yesterday + created_at: 1.day.ago ) } let(:event_one_day_outside_window) { Event.create!( identifier: "one_day_outside_window", payload: {name: "bax", value: 2}, - created_at: Date.yesterday - 1.day + created_at: 2.days.ago ) } let(:event_at_edge_of_window) { Event.create!( identifier: "at_edge_of_window", payload: {name: "bax", value: 2}, - created_at: Date.yesterday.midnight + created_at: 1.day.ago.midnight ) } let(:event_today) { Event.create!( identifier: "today", payload: {name: "bax", value: 2}, - created_at: Date.today + created_at: Time.now ) } @@ -343,28 +342,28 @@ Event.create!( identifier: "today", payload: {name: "bar", value: 2}, - created_at: Date.today + created_at: 1.minute.ago ) } let(:event_one_day_outside_window) { Event.create!( identifier: "one_day_outside_window", payload: {name: "bax", value: 2}, - created_at: Date.yesterday + created_at: 2.days.ago ) } let(:event_at_edge_of_window) { Event.create!( identifier: "at_edge_of_window", payload: {name: "bax", value: 2}, - created_at: Date.today.midnight + created_at: Time.now.end_of_day - 1.minute ) } it "returns all the records that were created today" do aggregate_failures do - expect(Event.today).to match_array([event_today, event_at_edge_of_window]) - expect(Event.today).not_to include(event_one_day_outside_window) + expect(Event.today).to match_array([event_today]) + expect(Event.today).not_to include(event_one_day_outside_window, event_at_edge_of_window) end end end diff --git a/spec/timescaledb/migration_helper_spec.rb b/spec/timescaledb/migration_helper_spec.rb index 0c0b0a7..be4a706 100644 --- a/spec/timescaledb/migration_helper_spec.rb +++ b/spec/timescaledb/migration_helper_spec.rb @@ -22,7 +22,7 @@ chunk_time_interval: '1 min', compress_segmentby: 'identifier', compress_orderby: 'created_at', - compression_interval: '7 days' + compress_after: '7 days' } end @@ -79,7 +79,7 @@ chunk_time_interval: '1 min', compress_segmentby: 'symbol', compress_orderby: 'created_at', - compression_interval: '7 days' + compress_after: '7 days' } end diff --git a/spec/timescaledb/toolkit_helper_spec.rb b/spec/timescaledb/toolkit_helper_spec.rb index e2ac4d7..d534b74 100644 --- a/spec/timescaledb/toolkit_helper_spec.rb +++ b/spec/timescaledb/toolkit_helper_spec.rb @@ -9,7 +9,7 @@ chunk_time_interval: '1 day', compress_segmentby: 'device_id', compress_orderby: 'ts', - compression_interval: '7 days' + compress_after: '7 days' } end