diff --git a/CHANGELOG.md b/CHANGELOG.md index c82eb648..51478758 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## UNRELEASED +# 1.18.3 - 2022-11-16 + +### Features :star: + +- Added ActiveRecordConsumerGenerator, which streamlines the process into a single flow +- Creates a database migration, Rails model, consumer, consumer config, and Deimos schema classes + # 1.18.2 - 2022-11-14 - Fixes bug related to wait between runs when no records are fetched by updating poll_info diff --git a/README.md b/README.md index fe74e7ad..21af6300 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Built on Phobos and hence Ruby-Kafka. * [Instrumentation](#instrumentation) * [Kafka Message Keys](#kafka-message-keys) * [Consumers](#consumers) + * [Active Record Consumer Generator](#active-record-consumer-generator) * [Rails Integration](#rails-integration) * [Controller Mixin](#controller-mixin) * [Database Backend](#database-backend) @@ -309,6 +310,49 @@ class MyConsumer < Deimos::Consumer end ``` +## Active Record Consumer Generator + +Deimos provides a generator that streamlines Deimos consumer creation. It +creates all the necessary components needed in a Ruby on Rails application when +creating a new Active Record Consumer. + +It will take an existing schema and generate the following files based on its fields: + + Database Migration + Rails Model + Consumer Class + Deimos Consumer Config + Generated Schema Classes + + +By default, any complex sub-types (such as records or arrays) are turned into JSON +(if supported) or string columns. + +Before running this generator, you must first copy the schema into your repo in the +correct path. + +To generate a migration, model, consumer class, consumer config and schema classes, run the following: + +Usage: + + rails g deimos:active_record_consumer FULL_SCHEMA_NAME [CONFIG_FILE_PATH] + + Options are... + FULL_SCHEMA_NAME (required) Fully qualified schema name, located in config.schema.path that defines the Avro schema for the Consumer. + CONFIG_FILE_PATH (optional) Path to existing config file. If not specified, defaults to config/initializers/deimos.rb. + +Example: + + rails g deimos:active_record_consumer com.my-namespace.Widget + +...would generate: + + db/migrate/20221111134112_create_widgets.rb + app/models/widget.rb + app/lib/kafka/models/widget_consumer.rb + config/initializers/deimos.rb + app/lib/schema_classes/**/*.rb + ### Fatal Errors The recommended configuration is for consumers *not* to raise errors diff --git a/lib/generators/deimos/active_record_consumer/USAGE b/lib/generators/deimos/active_record_consumer/USAGE new file mode 100644 index 00000000..d8245593 --- /dev/null +++ b/lib/generators/deimos/active_record_consumer/USAGE @@ -0,0 +1,23 @@ +Description: + Streamlines Deimos consumer creation. Creates all the necessary components needed in a Ruby on Rails application when creating a new Active Record Consumer. + +Usage: + bin/rails generate deimos:active_record_consumer FULL_SCHEMA_NAME [CONFIG_FILE_PATH] + + Options are... + FULL_SCHEMA_NAME (required) Fully qualified schema name, located in config.schema.path that defines the Avro schema for the Consumer. + CONFIG_FILE_PATH (optional) Path to existing config file. If not specified, defaults to config/initializers/deimos.rb. + +Example: + bin/rails generate deimos:active_record_consumer com.test.Widget config/initializers/myconfig.config + + This will create: + Database Migration db/migrate/20221111134112_create_widgets.rb + Rails Model app/models/widget.rb + Consumer Class app/lib/kafka/models/widget_consumer.rb + Deimos Consumer Config config/initializers/deimos.rb or adds config to CONFIG_FILE_PATH + Generated Schema Classes app/lib/schema_classes or config.schema.generated_class_path + + + + diff --git a/lib/generators/deimos/active_record_consumer/templates/config.rb.tt b/lib/generators/deimos/active_record_consumer/templates/config.rb.tt new file mode 100644 index 00000000..a89d0ad8 --- /dev/null +++ b/lib/generators/deimos/active_record_consumer/templates/config.rb.tt @@ -0,0 +1,4 @@ +Deimos.configure do +<%= @consumer_config %> +end + diff --git a/lib/generators/deimos/active_record_consumer/templates/consumer.rb.tt b/lib/generators/deimos/active_record_consumer/templates/consumer.rb.tt new file mode 100644 index 00000000..a0450258 --- /dev/null +++ b/lib/generators/deimos/active_record_consumer/templates/consumer.rb.tt @@ -0,0 +1,45 @@ +class <%= consumer_name.classify %> < Deimos::ActiveRecordConsumer + record_class <%= model_name.classify %> + + # Optional override of the way to fetch records based on payload and + # key. Default is to use the key to search the primary key of the table. + # Only used in non-batch mode. + def fetch_record(klass, payload, key) + super + end + + # Optional override on how to set primary key for new records. + # Default is to set the class's primary key to the message's decoded key. + # Only used in non-batch mode. + def assign_key(record, payload, key) + super + end + + # Optional override of the default behavior, which is to call `destroy` + # on the record - e.g. you can replace this with "archiving" the record + # in some way. + # Only used in non-batch mode. + def destroy_record(record) + super + end + + # Optional override to change the attributes of the record before they + # are saved. + def record_attributes(payload, key) + super.merge(:some_field => 'some_value') + end + + # Optional override to change the attributes used for identifying records + def record_key(payload) + super + end + + # Optional override, returns true by default. + # When this method returns true, a record corresponding to the message + # is created/updated. + # When this method returns false, message processing is skipped and a + # corresponding record will NOT be created/updated. + def process_message?(payload) + super + end +end diff --git a/lib/generators/deimos/active_record_consumer/templates/consumer_config.rb.tt b/lib/generators/deimos/active_record_consumer/templates/consumer_config.rb.tt new file mode 100644 index 00000000..ae8492c8 --- /dev/null +++ b/lib/generators/deimos/active_record_consumer/templates/consumer_config.rb.tt @@ -0,0 +1,12 @@ + # Define a consumer + consumer do + class_name '<%= consumer_name.classify %>' + topic 'TopicToConsume' + schema '<%= schema %>' + namespace '<%= namespace %>' + key_config <%= @key_type %>: :<%= @key_value %> + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end diff --git a/lib/generators/deimos/active_record_consumer/templates/migration.rb.tt b/lib/generators/deimos/active_record_consumer/templates/migration.rb.tt new file mode 100644 index 00000000..ce7c5454 --- /dev/null +++ b/lib/generators/deimos/active_record_consumer/templates/migration.rb.tt @@ -0,0 +1,30 @@ +class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %> + def up + if table_exists?(:<%= table_name %>) + warn "<%= table_name %> already exists, exiting" + return + end + create_table :<%= table_name %> do |t| + <%- fields.each do |key| -%> + <%- next if %w(id message_id timestamp updated_at created_at).include?(key.name) -%> + <%- sql_type = schema_base.sql_type(key) + if %w(record array map).include?(sql_type) + conn = ActiveRecord::Base.connection + sql_type = conn.respond_to?(:supports_json?) && conn.supports_json? ? :json : :string + end + -%> + t.<%= sql_type %> :<%= key.name %> + <%- end -%> + + t.timestamps + + # TODO add indexes as necessary + end + end + + def down + return unless table_exists?(:<%= table_name %>) + drop_table :<%= table_name %> + end + +end diff --git a/lib/generators/deimos/active_record_consumer/templates/model.rb.tt b/lib/generators/deimos/active_record_consumer/templates/model.rb.tt new file mode 100644 index 00000000..e40b23da --- /dev/null +++ b/lib/generators/deimos/active_record_consumer/templates/model.rb.tt @@ -0,0 +1,5 @@ +class <%= table_name.classify %> < ApplicationRecord +<%- fields.select { |f| f.enum_values.any? }.each do |field| -%> + enum <%= field.name %>: {<%= field.enum_values.map { |v| "#{v}: '#{v}'"}.join(', ') %>} +<% end -%> +end diff --git a/lib/generators/deimos/active_record_consumer_generator.rb b/lib/generators/deimos/active_record_consumer_generator.rb new file mode 100644 index 00000000..8c2c2b0f --- /dev/null +++ b/lib/generators/deimos/active_record_consumer_generator.rb @@ -0,0 +1,191 @@ +# frozen_string_literal: true + +require 'rails/generators' +require 'rails/generators/active_record/migration' +require 'generators/deimos/schema_class_generator' +require 'generators/deimos/active_record_generator' +require 'rails/version' +require 'erb' + +# Generates a new Active Record Consumer, as well +# as all the necessary files and configuration. +# Streamlines the creation process into a single flow. +module Deimos + module Generators + # Generator for ActiveRecordConsumer db migration, Rails model, Consumer, Consumer config + # and Deimos Schema class + class ActiveRecordConsumerGenerator < Rails::Generators::Base + include Rails::Generators::Migration + if Rails.version < '4' + extend(ActiveRecord::Generators::Migration) + else + include ActiveRecord::Generators::Migration + end + + # @return [Array] + KEY_CONFIG_OPTIONS = %w(none plain schema field).freeze + + # @return [Array] + KEY_CONFIG_OPTIONS_BOOL = %w(none plain).freeze + + # @return [Array] + KEY_CONFIG_OPTIONS_STRING = %w(schema field).freeze + + source_root File.expand_path('active_record_consumer/templates', __dir__) + + argument :full_schema, desc: 'The fully qualified schema name.', required: true + argument :key_config_type, desc: 'The kafka message key configuration type.', required: true + argument :key_config_value, desc: 'The kafka message key configuration value.', required: true + argument :config_path, desc: 'The path to the deimos configuration file, relative to the root directory.', required: false + + no_commands do + + # validate schema, key_config and deimos config file + def validate_arguments + _validate_schema + _validate_key_config + _validate_config_path + end + + # Creates database migration for creating new table and Rails Model + def create_db_migration_rails_model + Deimos::Generators::ActiveRecordGenerator.start([table_name,full_schema]) + end + + # Creates Kafka Consumer file + def create_consumer + template('consumer.rb', "app/lib/kafka/models/#{consumer_name.underscore}.rb") + end + + # Adds consumer config to config file. + # Defaults to deimos.rb if config_path arg is not specified. + def create_consumer_config + if @config_file_path.nil? + config_file = 'deimos.rb' + @config_file_path = "#{initializer_path}/#{config_file}" + end + + consumer_config_template = File.expand_path(find_in_source_paths('consumer_config.rb')) + if File.exist?(@config_file_path) + # if file has Deimos.configure statement then add consumers block after it + if File.readlines(@config_file_path).grep(/Deimos.configure do/).size > 0 + insert_into_file(@config_file_path.to_s, + CapturableERB.new(::File.binread(consumer_config_template)).result(binding), + :after => "Deimos.configure do\n") + else + # if file does not have Deimos.configure statement then add it plus consumers block + @consumer_config = CapturableERB.new(::File.binread(consumer_config_template)).result(binding) + config_template = File.expand_path(find_in_source_paths('config.rb')) + insert_into_file(@config_file_path.to_s,CapturableERB.new(::File.binread(config_template)).result(binding)) + end + else + @consumer_config = CapturableERB.new(::File.binread(consumer_config_template)).result(binding) + template('config.rb', @config_file_path.to_s) + end + end + + # Generates schema classes + def create_deimos_schema_class + Deimos::Generators::SchemaClassGenerator.start(['--skip_generate_from_schema_files']) + end + + # @return [String] + def schema + last_dot = self.full_schema.rindex('.') + self.full_schema[last_dot + 1..-1] + end + + # @return [String] + def namespace + last_dot = self.full_schema.rindex('.') + self.full_schema[0...last_dot] + end + + # @return [Deimos::SchemaBackends::Base] + def schema_base + @schema_base ||= Deimos.schema_backend_class.new(schema: schema, namespace: namespace) + end + + # @return [Array] + def fields + schema_base.schema_fields + end + + # @return [String] + def table_name + schema.tableize + end + + # @return [String] + def model_name + table_name.underscore.singularize + end + + # @return [String] + def consumer_name + "#{schema.classify}Consumer" + end + + # @return [String] + def initializer_path + if defined?(Rails.application) && Rails.application + paths = Rails.application.config.paths['config/initializers'] + paths.respond_to?(:to_ary) ? paths.to_ary.first : paths.to_a.first + else + 'config/initializers' + end + end + + # # @return [String] Returns the name of the first field in the schema, as the key + # def key_field + # fields.first.name + # end + end + + # desc 'Generate necessary files and configuration for a new Active Record Consumer.' + # @return [void] + def generate + # if yes?('Would you like to install Rspec?') + # gem 'rspec-rails', group: :test + # after_bundle { generate 'rspec:install' } + # end + validate_arguments + create_db_migration_rails_model + create_consumer + create_consumer_config + create_deimos_schema_class + end + + private + + # Determines if Schema Class Generation can be run. + # @raise if Schema Backend is not of a Avro-based class + def _validate_schema + backend = Deimos.config.schema.backend.to_s + raise 'Schema Class Generation requires an Avro-based Schema Backend' if backend !~ /^avro/ + end + +# key_config none: true - this indicates that you are not using keys at all for this topic. This must be set if your messages won't have keys - either all your messages in a topic need to have a key, or they all need to have no key. This is a good choice for events that aren't keyed - you can still set a partition key. +# key_config plain: true - this indicates that you are not using an encoded key. Use this for legacy topics - new topics should not use this setting. +# key_config schema: 'MyKeySchema-key' - this tells the producer to look for an existing key schema named MyKeySchema-key in the schema registry and to encode the key using it. Use this if you've already created a key schema or the key value does not exist in the existing payload (e.g. it is a compound or generated key). +# key_config field: 'my_field' - this tells the producer to look for a field named my_field in the value schema. When a payload comes in, the producer will take that value from the payload and insert it in a dynamically generated key schema. This key schema does not need to live in your codebase. Instead, it will be a subset of the value schema with only the key field in it. + def _validate_key_config + @key_type = key_config_type + if KEY_CONFIG_OPTIONS_BOOL.include?(key_config_type) + @key_value = 'true' + elsif KEY_CONFIG_OPTIONS_STRING.include?(key_config_type) + @key_value = key_config_value + else + raise 'Invalid key config specified!' + end + end + + def _validate_config_path + if config_path.present? + @config_file_path = "#{initializer_path}/#{config_path}" + raise 'Configuration file does not exist!' unless File.exist?(@config_file_path) + end + end + end + end +end diff --git a/lib/generators/deimos/schema_class_generator.rb b/lib/generators/deimos/schema_class_generator.rb index 0e30541e..bc3891a8 100644 --- a/lib/generators/deimos/schema_class_generator.rb +++ b/lib/generators/deimos/schema_class_generator.rb @@ -26,6 +26,8 @@ class SchemaClassGenerator < Rails::Generators::Base source_root File.expand_path('schema_class/templates', __dir__) + class_option :skip_generate_from_schema_files, type: :boolean, default: false + no_commands do # Retrieve the fields from this Avro Schema # @param schema [Avro::Schema::NamedSchema] @@ -175,7 +177,7 @@ def generate generate_classes(schema_name, namespace, config.key_config) end - generate_from_schema_files(found_schemas) + generate_from_schema_files(found_schemas) unless options[:skip_generate_from_schema_files] end diff --git a/spec/generators/active_record_consumer_generator_spec.rb b/spec/generators/active_record_consumer_generator_spec.rb new file mode 100644 index 00000000..25d3a167 --- /dev/null +++ b/spec/generators/active_record_consumer_generator_spec.rb @@ -0,0 +1,241 @@ +# frozen_string_literal: true + +require 'generators/deimos/active_record_consumer_generator' +require 'fileutils' +require 'tempfile' + +RSpec.describe Deimos::Generators::ActiveRecordConsumerGenerator do + let(:db_migration_path) { 'db/migrate' } + let(:model_path) { 'app/models' } + let(:consumer_path) { 'app/lib/kafka/models' } + let(:config_path) { 'config/initializers' } + let(:schema_class_path) { 'spec/app/lib/schema_classes' } + + after(:each) do + FileUtils.rm_rf('db') if File.exist?('db') + FileUtils.rm_rf('app') if File.exist?('app') + FileUtils.rm_rf('config') if File.exist?('config') + FileUtils.rm_rf('spec/app') if File.exist?('spec/app') + end + + before(:each) do + Deimos.config.reset! + Deimos.configure do + schema.path('spec/schemas/') + schema.generated_class_path('spec/app/lib/schema_classes') + schema.backend(:avro_local) + schema.generate_namespace_folders(true) + end + end + + context 'with a regular flow' do + + it 'should generate a migration, model, consumer class, config and schema class with existing deimos.rb config file' do + config_file_name = 'deimos.rb' + FileUtils.mkdir_p(config_path) + deimos_file_path = "#{config_path}/#{config_file_name}" + File.new(deimos_file_path, "w") + File.open(deimos_file_path, "w") { |f| f.write "Deimos.configure do\n\nend" } + + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end + end + + expect(Dir["#{db_migration_path}/*.rb"]).to be_empty + expect(Dir["#{model_path}/*.rb"]).to be_empty + expect(Dir["#{schema_class_path}/*.rb"]).to be_empty + + described_class.start(['com.my-namespace.Widget','field','a_string']) + + files = Dir["#{db_migration_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_migration') + + files = Dir["#{model_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_model') + + files = Dir["#{consumer_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_consumer_class') + + files = Dir["#{config_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_existing_deimos_config') + + files = Dir["#{schema_class_path}/*/*.rb"] + expect(File.read(files[0])).to match_snapshot('consumer_generator_schema_classes') + end + + end + + + context 'with different config file input' do + it 'should generate correct config with no existing deimos.rb configuration file' do + + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end + end + + expect(Dir["#{db_migration_path}/*.rb"]).to be_empty + expect(Dir["#{model_path}/*.rb"]).to be_empty + expect(Dir["#{schema_class_path}/*.rb"]).to be_empty + + described_class.start(['com.my-namespace.Widget','field','a-string']) + + files = Dir["#{config_path}/deimos.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_new_deimos_config') + end + + it 'should generate correct config for the passed in config file that contains Deimos.configure' do + config_file_name = 'my_config.config' + FileUtils.mkdir_p(config_path) + my_config_file_path = "#{config_path}/#{config_file_name}" + File.new(my_config_file_path, "w") + File.open(my_config_file_path, "w") { |f| f.write "Deimos.configure do\n\nend" } + + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end + end + + expect(Dir["#{db_migration_path}/*.rb"]).to be_empty + expect(Dir["#{model_path}/*.rb"]).to be_empty + expect(Dir["#{schema_class_path}/*.rb"]).to be_empty + + described_class.start(['com.my-namespace.Widget','field','a-string',config_file_name]) + + files = Dir["#{config_path}/*.config"] + expect(files.length).to eq(1) + expect(File.read(my_config_file_path)).to match_snapshot('consumer_generator_config_arg') + end + + it 'should generate correct config for the passed in config file with missing Deimos.configure' do + config_file_name = 'my_config.config' + FileUtils.mkdir_p(config_path) + my_config_file_path = "#{config_path}/#{config_file_name}" + File.new(my_config_file_path, "w") + + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end + end + + expect(Dir["#{db_migration_path}/*.rb"]).to be_empty + expect(Dir["#{model_path}/*.rb"]).to be_empty + expect(Dir["#{schema_class_path}/*.rb"]).to be_empty + + described_class.start(['com.my-namespace.Widget','field','a-string',config_file_name]) + + files = Dir["#{config_path}/*.config"] + expect(files.length).to eq(1) + expect(File.read(my_config_file_path)).to match_snapshot('consumer_generator_config_arg_missing_deimos_configure') + end + end + + context 'with different key schema arguments specified' do + it 'should generate correct key_config when specifying key_config none: true' do + config_file_name = 'my_config.config' + FileUtils.mkdir_p(config_path) + my_config_file_path = "#{config_path}/#{config_file_name}" + File.new(my_config_file_path, "w") + + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end + end + + expect(Dir["#{db_migration_path}/*.rb"]).to be_empty + expect(Dir["#{model_path}/*.rb"]).to be_empty + expect(Dir["#{schema_class_path}/*.rb"]).to be_empty + + described_class.start(['com.my-namespace.Widget','none','true',config_file_name]) + + files = Dir["#{config_path}/*.config"] + expect(files.length).to eq(1) + expect(File.read(my_config_file_path)).to match_snapshot('consumer_generator_key_config_none') + end + + it 'should generate correct key_config when specifying key_config plain: true' do + config_file_name = 'my_config.config' + FileUtils.mkdir_p(config_path) + my_config_file_path = "#{config_path}/#{config_file_name}" + File.new(my_config_file_path, "w") + + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end + end + + expect(Dir["#{db_migration_path}/*.rb"]).to be_empty + expect(Dir["#{model_path}/*.rb"]).to be_empty + expect(Dir["#{schema_class_path}/*.rb"]).to be_empty + + described_class.start(['com.my-namespace.Widget','plain','true',config_file_name]) + + files = Dir["#{config_path}/*.config"] + expect(files.length).to eq(1) + expect(File.read(my_config_file_path)).to match_snapshot('consumer_generator_key_config_plain') + end + + it 'should generate correct key_config when specifying key_config schema: MyKeySchema-key' do + config_file_name = 'my_config.config' + FileUtils.mkdir_p(config_path) + my_config_file_path = "#{config_path}/#{config_file_name}" + File.new(my_config_file_path, "w") + + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end + end + + expect(Dir["#{db_migration_path}/*.rb"]).to be_empty + expect(Dir["#{model_path}/*.rb"]).to be_empty + expect(Dir["#{schema_class_path}/*.rb"]).to be_empty + + described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key',config_file_name]) + + files = Dir["#{config_path}/*.config"] + expect(files.length).to eq(1) + expect(File.read(my_config_file_path)).to match_snapshot('consumer_generator_key_config_schema') + end + end +end diff --git a/spec/generators/schema_class_generator_spec.rb b/spec/generators/schema_class_generator_spec.rb index 00c87437..904f0140 100644 --- a/spec/generators/schema_class_generator_spec.rb +++ b/spec/generators/schema_class_generator_spec.rb @@ -255,4 +255,25 @@ def dump(value) end end + context 'with skip_generate_from_schema_files option' do + before(:each) do + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end + end + end + + it 'should only generate class for consumer defined in config' do + Deimos.with_config('schema.nest_child_schemas' => false) do + described_class.start(['--skip_generate_from_schema_files']) + expect(files).to match_snapshot('widget-skip-generate-from-schema-files', snapshot_serializer: MultiFileSerializer) + end + end + end + end diff --git a/spec/snapshots/consumer_generator_config_arg.snap b/spec/snapshots/consumer_generator_config_arg.snap new file mode 100644 index 00000000..c8fc0481 --- /dev/null +++ b/spec/snapshots/consumer_generator_config_arg.snap @@ -0,0 +1,15 @@ +Deimos.configure do + # Define a consumer + consumer do + class_name 'WidgetConsumer' + topic 'TopicToConsume' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a-string + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end + +end \ No newline at end of file diff --git a/spec/snapshots/consumer_generator_config_arg_missing_deimos_configure.snap b/spec/snapshots/consumer_generator_config_arg_missing_deimos_configure.snap new file mode 100644 index 00000000..c132e405 --- /dev/null +++ b/spec/snapshots/consumer_generator_config_arg_missing_deimos_configure.snap @@ -0,0 +1,16 @@ +Deimos.configure do + # Define a consumer + consumer do + class_name 'WidgetConsumer' + topic 'TopicToConsume' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a-string + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end + +end + diff --git a/spec/snapshots/consumer_generator_consumer_class.snap b/spec/snapshots/consumer_generator_consumer_class.snap new file mode 100644 index 00000000..adcd5cd4 --- /dev/null +++ b/spec/snapshots/consumer_generator_consumer_class.snap @@ -0,0 +1,45 @@ +class WidgetConsumer < Deimos::ActiveRecordConsumer + record_class Widget + + # Optional override of the way to fetch records based on payload and + # key. Default is to use the key to search the primary key of the table. + # Only used in non-batch mode. + def fetch_record(klass, payload, key) + super + end + + # Optional override on how to set primary key for new records. + # Default is to set the class's primary key to the message's decoded key. + # Only used in non-batch mode. + def assign_key(record, payload, key) + super + end + + # Optional override of the default behavior, which is to call `destroy` + # on the record - e.g. you can replace this with "archiving" the record + # in some way. + # Only used in non-batch mode. + def destroy_record(record) + super + end + + # Optional override to change the attributes of the record before they + # are saved. + def record_attributes(payload, key) + super.merge(:some_field => 'some_value') + end + + # Optional override to change the attributes used for identifying records + def record_key(payload) + super + end + + # Optional override, returns true by default. + # When this method returns true, a record corresponding to the message + # is created/updated. + # When this method returns false, message processing is skipped and a + # corresponding record will NOT be created/updated. + def process_message?(payload) + super + end +end diff --git a/spec/snapshots/consumer_generator_existing_deimos_config.snap b/spec/snapshots/consumer_generator_existing_deimos_config.snap new file mode 100644 index 00000000..c8cc12b2 --- /dev/null +++ b/spec/snapshots/consumer_generator_existing_deimos_config.snap @@ -0,0 +1,15 @@ +Deimos.configure do + # Define a consumer + consumer do + class_name 'WidgetConsumer' + topic 'TopicToConsume' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end + +end \ No newline at end of file diff --git a/spec/snapshots/consumer_generator_key_config_none.snap b/spec/snapshots/consumer_generator_key_config_none.snap new file mode 100644 index 00000000..fc37d5a2 --- /dev/null +++ b/spec/snapshots/consumer_generator_key_config_none.snap @@ -0,0 +1,16 @@ +Deimos.configure do + # Define a consumer + consumer do + class_name 'WidgetConsumer' + topic 'TopicToConsume' + schema 'Widget' + namespace 'com.my-namespace' + key_config none: :true + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end + +end + diff --git a/spec/snapshots/consumer_generator_key_config_plain.snap b/spec/snapshots/consumer_generator_key_config_plain.snap new file mode 100644 index 00000000..d210c7c6 --- /dev/null +++ b/spec/snapshots/consumer_generator_key_config_plain.snap @@ -0,0 +1,16 @@ +Deimos.configure do + # Define a consumer + consumer do + class_name 'WidgetConsumer' + topic 'TopicToConsume' + schema 'Widget' + namespace 'com.my-namespace' + key_config plain: :true + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end + +end + diff --git a/spec/snapshots/consumer_generator_key_config_schema.snap b/spec/snapshots/consumer_generator_key_config_schema.snap new file mode 100644 index 00000000..4c703e67 --- /dev/null +++ b/spec/snapshots/consumer_generator_key_config_schema.snap @@ -0,0 +1,16 @@ +Deimos.configure do + # Define a consumer + consumer do + class_name 'WidgetConsumer' + topic 'TopicToConsume' + schema 'Widget' + namespace 'com.my-namespace' + key_config schema: :MyKeySchema-key + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end + +end + diff --git a/spec/snapshots/consumer_generator_migration.snap b/spec/snapshots/consumer_generator_migration.snap new file mode 100644 index 00000000..4689489e --- /dev/null +++ b/spec/snapshots/consumer_generator_migration.snap @@ -0,0 +1,22 @@ +class CreateWidgets < ActiveRecord::Migration[6.1] + def up + if table_exists?(:widgets) + warn "widgets already exists, exiting" + return + end + create_table :widgets do |t| + t.bigint :widget_id + t.string :name + + t.timestamps + + # TODO add indexes as necessary + end + end + + def down + return unless table_exists?(:widgets) + drop_table :widgets + end + +end diff --git a/spec/snapshots/consumer_generator_model.snap b/spec/snapshots/consumer_generator_model.snap new file mode 100644 index 00000000..a523af0a --- /dev/null +++ b/spec/snapshots/consumer_generator_model.snap @@ -0,0 +1,2 @@ +class Widget < ApplicationRecord +end diff --git a/spec/snapshots/consumer_generator_new_deimos_config.snap b/spec/snapshots/consumer_generator_new_deimos_config.snap new file mode 100644 index 00000000..c132e405 --- /dev/null +++ b/spec/snapshots/consumer_generator_new_deimos_config.snap @@ -0,0 +1,16 @@ +Deimos.configure do + # Define a consumer + consumer do + class_name 'WidgetConsumer' + topic 'TopicToConsume' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a-string + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end + +end + diff --git a/spec/snapshots/consumer_generator_schema_classes.snap b/spec/snapshots/consumer_generator_schema_classes.snap new file mode 100644 index 00000000..614a0310 --- /dev/null +++ b/spec/snapshots/consumer_generator_schema_classes.snap @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module MyNamespace + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.Widget + class Widget < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [Integer] + attr_accessor :id + # @return [Integer] + attr_accessor :widget_id + # @return [String] + attr_accessor :name + # @return [Integer] + attr_accessor :updated_at + # @return [Integer] + attr_accessor :created_at + + # @override + def initialize(id: nil, + widget_id: nil, + name: nil, + updated_at: nil, + created_at: nil) + super + self.id = id + self.widget_id = widget_id + self.name = name + self.updated_at = updated_at + self.created_at = created_at + end + + # @override + def schema + 'Widget' + end + + # @override + def namespace + 'com.my-namespace' + end + + def self.tombstone(key) + record = self.allocate + record.tombstone_key = key + record.a_string = key + record + end + + # @override + def as_json(_opts={}) + { + 'id' => @id, + 'widget_id' => @widget_id, + 'name' => @name, + 'updated_at' => @updated_at, + 'created_at' => @created_at + } + end + end +end; end diff --git a/spec/snapshots/widget-skip-generate-from-schema-files.snap b/spec/snapshots/widget-skip-generate-from-schema-files.snap new file mode 100644 index 00000000..736c0df4 --- /dev/null +++ b/spec/snapshots/widget-skip-generate-from-schema-files.snap @@ -0,0 +1,65 @@ +spec/app/lib/schema_classes/widget.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.Widget + class Widget < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [Integer] + attr_accessor :id + # @return [Integer] + attr_accessor :widget_id + # @return [String] + attr_accessor :name + # @return [Integer] + attr_accessor :updated_at + # @return [Integer] + attr_accessor :created_at + + # @override + def initialize(id: nil, + widget_id: nil, + name: nil, + updated_at: nil, + created_at: nil) + super + self.id = id + self.widget_id = widget_id + self.name = name + self.updated_at = updated_at + self.created_at = created_at + end + + # @override + def schema + 'Widget' + end + + # @override + def namespace + 'com.my-namespace' + end + + def self.tombstone(key) + record = self.allocate + record.tombstone_key = key + record.a_string = key + record + end + + # @override + def as_json(_opts={}) + { + 'id' => @id, + 'widget_id' => @widget_id, + 'name' => @name, + 'updated_at' => @updated_at, + 'created_at' => @created_at + } + end + end +end +