From 65c8d5de49d5b895575c62670ae12f2a25b4a160 Mon Sep 17 00:00:00 2001 From: jenchanflipp Date: Tue, 15 Nov 2022 16:38:20 -0500 Subject: [PATCH 1/7] GUILD-690: Initial draft of new Active Record Consumer Generator --- README.md | 44 + .../deimos/active_record_consumer/USAGE | 23 + .../templates/config.rb.tt | 29 + .../templates/consumer.rb.tt | 45 + .../templates/migration.rb.tt | 30 + .../templates/model.rb.tt | 5 + .../active_record_consumer_generator.rb | 151 ++ spec/snapshots/consumer_generator_config.snap | 29 + ...umer_generator_config_with_config_arg.snap | 29 + .../consumer_generator_consumer_class.snap | 45 + .../consumer_generator_migration.snap | 22 + spec/snapshots/consumer_generator_model.snap | 2 + .../consumer_generator_schema_classes.snap | 1379 +++++++++++++++++ 13 files changed, 1833 insertions(+) create mode 100644 lib/generators/deimos/active_record_consumer/USAGE create mode 100644 lib/generators/deimos/active_record_consumer/templates/config.rb.tt create mode 100644 lib/generators/deimos/active_record_consumer/templates/consumer.rb.tt create mode 100644 lib/generators/deimos/active_record_consumer/templates/migration.rb.tt create mode 100644 lib/generators/deimos/active_record_consumer/templates/model.rb.tt create mode 100644 lib/generators/deimos/active_record_consumer_generator.rb create mode 100644 spec/snapshots/consumer_generator_config.snap create mode 100644 spec/snapshots/consumer_generator_config_with_config_arg.snap create mode 100644 spec/snapshots/consumer_generator_consumer_class.snap create mode 100644 spec/snapshots/consumer_generator_migration.snap create mode 100644 spec/snapshots/consumer_generator_model.snap create mode 100644 spec/snapshots/consumer_generator_schema_classes.snap diff --git a/README.md b/README.md index fe74e7ad..21af6300 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Built on Phobos and hence Ruby-Kafka. * [Instrumentation](#instrumentation) * [Kafka Message Keys](#kafka-message-keys) * [Consumers](#consumers) + * [Active Record Consumer Generator](#active-record-consumer-generator) * [Rails Integration](#rails-integration) * [Controller Mixin](#controller-mixin) * [Database Backend](#database-backend) @@ -309,6 +310,49 @@ class MyConsumer < Deimos::Consumer end ``` +## Active Record Consumer Generator + +Deimos provides a generator that streamlines Deimos consumer creation. It +creates all the necessary components needed in a Ruby on Rails application when +creating a new Active Record Consumer. + +It will take an existing schema and generate the following files based on its fields: + + Database Migration + Rails Model + Consumer Class + Deimos Consumer Config + Generated Schema Classes + + +By default, any complex sub-types (such as records or arrays) are turned into JSON +(if supported) or string columns. + +Before running this generator, you must first copy the schema into your repo in the +correct path. + +To generate a migration, model, consumer class, consumer config and schema classes, run the following: + +Usage: + + rails g deimos:active_record_consumer FULL_SCHEMA_NAME [CONFIG_FILE_PATH] + + Options are... + FULL_SCHEMA_NAME (required) Fully qualified schema name, located in config.schema.path that defines the Avro schema for the Consumer. + CONFIG_FILE_PATH (optional) Path to existing config file. If not specified, defaults to config/initializers/deimos.rb. + +Example: + + rails g deimos:active_record_consumer com.my-namespace.Widget + +...would generate: + + db/migrate/20221111134112_create_widgets.rb + app/models/widget.rb + app/lib/kafka/models/widget_consumer.rb + config/initializers/deimos.rb + app/lib/schema_classes/**/*.rb + ### Fatal Errors The recommended configuration is for consumers *not* to raise errors diff --git a/lib/generators/deimos/active_record_consumer/USAGE b/lib/generators/deimos/active_record_consumer/USAGE new file mode 100644 index 00000000..d8245593 --- /dev/null +++ b/lib/generators/deimos/active_record_consumer/USAGE @@ -0,0 +1,23 @@ +Description: + Streamlines Deimos consumer creation. Creates all the necessary components needed in a Ruby on Rails application when creating a new Active Record Consumer. + +Usage: + bin/rails generate deimos:active_record_consumer FULL_SCHEMA_NAME [CONFIG_FILE_PATH] + + Options are... + FULL_SCHEMA_NAME (required) Fully qualified schema name, located in config.schema.path that defines the Avro schema for the Consumer. + CONFIG_FILE_PATH (optional) Path to existing config file. If not specified, defaults to config/initializers/deimos.rb. + +Example: + bin/rails generate deimos:active_record_consumer com.test.Widget config/initializers/myconfig.config + + This will create: + Database Migration db/migrate/20221111134112_create_widgets.rb + Rails Model app/models/widget.rb + Consumer Class app/lib/kafka/models/widget_consumer.rb + Deimos Consumer Config config/initializers/deimos.rb or adds config to CONFIG_FILE_PATH + Generated Schema Classes app/lib/schema_classes or config.schema.generated_class_path + + + + diff --git a/lib/generators/deimos/active_record_consumer/templates/config.rb.tt b/lib/generators/deimos/active_record_consumer/templates/config.rb.tt new file mode 100644 index 00000000..f507161f --- /dev/null +++ b/lib/generators/deimos/active_record_consumer/templates/config.rb.tt @@ -0,0 +1,29 @@ + +Deimos.configure do + logger Logger.new(STDOUT) + # Nested config field + kafka.seed_brokers ['my.kafka.broker:9092'] + + schema.generate_namespace_folders true + schema.generated_class_path 'app/lib/schemas' + + # Multiple nested config fields via block + consumers do + session_timeout 30 + offset_commit_interval 10 + end + + # Define a consumer + consumer do + class_name '<%= consumer_name.classify %>' + topic 'TopicToConsume' + schema '<%= schema %>' + namespace '<%= namespace %>' + key_config field: :<%= key_field %> + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end + +end diff --git a/lib/generators/deimos/active_record_consumer/templates/consumer.rb.tt b/lib/generators/deimos/active_record_consumer/templates/consumer.rb.tt new file mode 100644 index 00000000..a0450258 --- /dev/null +++ b/lib/generators/deimos/active_record_consumer/templates/consumer.rb.tt @@ -0,0 +1,45 @@ +class <%= consumer_name.classify %> < Deimos::ActiveRecordConsumer + record_class <%= model_name.classify %> + + # Optional override of the way to fetch records based on payload and + # key. Default is to use the key to search the primary key of the table. + # Only used in non-batch mode. + def fetch_record(klass, payload, key) + super + end + + # Optional override on how to set primary key for new records. + # Default is to set the class's primary key to the message's decoded key. + # Only used in non-batch mode. + def assign_key(record, payload, key) + super + end + + # Optional override of the default behavior, which is to call `destroy` + # on the record - e.g. you can replace this with "archiving" the record + # in some way. + # Only used in non-batch mode. + def destroy_record(record) + super + end + + # Optional override to change the attributes of the record before they + # are saved. + def record_attributes(payload, key) + super.merge(:some_field => 'some_value') + end + + # Optional override to change the attributes used for identifying records + def record_key(payload) + super + end + + # Optional override, returns true by default. + # When this method returns true, a record corresponding to the message + # is created/updated. + # When this method returns false, message processing is skipped and a + # corresponding record will NOT be created/updated. + def process_message?(payload) + super + end +end diff --git a/lib/generators/deimos/active_record_consumer/templates/migration.rb.tt b/lib/generators/deimos/active_record_consumer/templates/migration.rb.tt new file mode 100644 index 00000000..ce7c5454 --- /dev/null +++ b/lib/generators/deimos/active_record_consumer/templates/migration.rb.tt @@ -0,0 +1,30 @@ +class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %> + def up + if table_exists?(:<%= table_name %>) + warn "<%= table_name %> already exists, exiting" + return + end + create_table :<%= table_name %> do |t| + <%- fields.each do |key| -%> + <%- next if %w(id message_id timestamp updated_at created_at).include?(key.name) -%> + <%- sql_type = schema_base.sql_type(key) + if %w(record array map).include?(sql_type) + conn = ActiveRecord::Base.connection + sql_type = conn.respond_to?(:supports_json?) && conn.supports_json? ? :json : :string + end + -%> + t.<%= sql_type %> :<%= key.name %> + <%- end -%> + + t.timestamps + + # TODO add indexes as necessary + end + end + + def down + return unless table_exists?(:<%= table_name %>) + drop_table :<%= table_name %> + end + +end diff --git a/lib/generators/deimos/active_record_consumer/templates/model.rb.tt b/lib/generators/deimos/active_record_consumer/templates/model.rb.tt new file mode 100644 index 00000000..e40b23da --- /dev/null +++ b/lib/generators/deimos/active_record_consumer/templates/model.rb.tt @@ -0,0 +1,5 @@ +class <%= table_name.classify %> < ApplicationRecord +<%- fields.select { |f| f.enum_values.any? }.each do |field| -%> + enum <%= field.name %>: {<%= field.enum_values.map { |v| "#{v}: '#{v}'"}.join(', ') %>} +<% end -%> +end diff --git a/lib/generators/deimos/active_record_consumer_generator.rb b/lib/generators/deimos/active_record_consumer_generator.rb new file mode 100644 index 00000000..f0d62b20 --- /dev/null +++ b/lib/generators/deimos/active_record_consumer_generator.rb @@ -0,0 +1,151 @@ +# frozen_string_literal: true + +require 'rails/generators' +require 'rails/generators/active_record/migration' +require 'generators/deimos/schema_class_generator' +require 'rails/version' +require 'erb' + +# Generates a new Active Record Consumer, as well +# as all the necessary files and configuration. +# Streamlines the creation process into a single flow. +module Deimos + module Generators + # Generator for ActiveRecordConsumer db migration, Rails model, Consumer, Consumer config + # and Deimos Schema class + class ActiveRecordConsumerGenerator < Rails::Generators::Base + include Rails::Generators::Migration + if Rails.version < '4' + extend(ActiveRecord::Generators::Migration) + else + include ActiveRecord::Generators::Migration + end + source_root File.expand_path('active_record_consumer/templates', __dir__) + + argument :full_schema, desc: 'The fully qualified schema name.', required: true + argument :config_path, desc: 'The path to the deimos configuration file, relative to the root directory.', required: false + + no_commands do + + # Creates database migration for creating new table + def create_db_migration + migration_template('migration.rb', "#{db_migrate_path}/create_#{table_name.underscore}.rb") + end + + # Creates Rails Model + def create_rails_model + template('model.rb', "app/models/#{model_name}.rb") + end + + # Creates Kafka Consumer file + def create_consumer + template('consumer.rb', "app/lib/kafka/models/#{consumer_name.underscore}.rb") + end + + # Adds consumer config to config file. + # Defaults to deimos.rb if config_path arg is not specified. + def create_consumer_config + config_file = 'deimos.rb' + config_file_path = "#{initializer_path}/#{config_file}" + config_file_path = config_path if config_path.present? + + if File.exist?(config_file_path) + config_template = File.expand_path(find_in_source_paths('config.rb')) + insert_into_file(config_file_path.to_s, CapturableERB.new(::File.binread(config_template)).result(binding)) + else + template('config.rb', config_file_path.to_s) + end + end + + # Generates schema classes + def create_deimos_schema_class + Deimos::Generators::SchemaClassGenerator.start + end + + # @return [String] + def db_migrate_path + if defined?(Rails.application) && Rails.application + paths = Rails.application.config.paths['db/migrate'] + paths.respond_to?(:to_ary) ? paths.to_ary.first : paths.to_a.first + else + 'db/migrate' + end + end + + # @return [String] + def migration_version + "[#{ActiveRecord::Migration.current_version}]" + rescue StandardError + '' + end + + # @return [String] + def table_class + table_name.classify + end + + # @return [String] + def schema + last_dot = self.full_schema.rindex('.') + self.full_schema[last_dot + 1..-1] + end + + # @return [String] + def namespace + last_dot = self.full_schema.rindex('.') + self.full_schema[0...last_dot] + end + + # @return [Deimos::SchemaBackends::Base] + def schema_base + @schema_base ||= Deimos.schema_backend_class.new(schema: schema, namespace: namespace) + end + + # @return [Array] + def fields + schema_base.schema_fields + end + + # @return [String] + def table_name + schema.tableize + end + + # @return [String] + def model_name + table_name.underscore.singularize + end + + # @return [String] + def consumer_name + "#{schema.classify}Consumer" + end + + # @return [String] + def initializer_path + if defined?(Rails.application) && Rails.application + paths = Rails.application.config.paths['config/initializers'] + paths.respond_to?(:to_ary) ? paths.to_ary.first : paths.to_a.first + else + 'config/initializers' + end + end + + # @return [String] Returns the name of the first field in the schema, as the key + def key_field + fields.first.name + end + end + + # desc 'Generate necessary files and configuration for a new Active Record Consumer.' + # @return [void] + def generate + create_db_migration + create_rails_model + create_consumer + create_consumer_config + create_deimos_schema_class + end + end + end +end diff --git a/spec/snapshots/consumer_generator_config.snap b/spec/snapshots/consumer_generator_config.snap new file mode 100644 index 00000000..40ed5d45 --- /dev/null +++ b/spec/snapshots/consumer_generator_config.snap @@ -0,0 +1,29 @@ + +Deimos.configure do + logger Logger.new(STDOUT) + # Nested config field + kafka.seed_brokers ['my.kafka.broker:9092'] + + schema.generate_namespace_folders true + schema.generated_class_path 'app/lib/schemas' + + # Multiple nested config fields via block + consumers do + session_timeout 30 + offset_commit_interval 10 + end + + # Define a consumer + consumer do + class_name 'WidgetConsumer' + topic 'TopicToConsume' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :id + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end + +end diff --git a/spec/snapshots/consumer_generator_config_with_config_arg.snap b/spec/snapshots/consumer_generator_config_with_config_arg.snap new file mode 100644 index 00000000..40ed5d45 --- /dev/null +++ b/spec/snapshots/consumer_generator_config_with_config_arg.snap @@ -0,0 +1,29 @@ + +Deimos.configure do + logger Logger.new(STDOUT) + # Nested config field + kafka.seed_brokers ['my.kafka.broker:9092'] + + schema.generate_namespace_folders true + schema.generated_class_path 'app/lib/schemas' + + # Multiple nested config fields via block + consumers do + session_timeout 30 + offset_commit_interval 10 + end + + # Define a consumer + consumer do + class_name 'WidgetConsumer' + topic 'TopicToConsume' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :id + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end + +end diff --git a/spec/snapshots/consumer_generator_consumer_class.snap b/spec/snapshots/consumer_generator_consumer_class.snap new file mode 100644 index 00000000..adcd5cd4 --- /dev/null +++ b/spec/snapshots/consumer_generator_consumer_class.snap @@ -0,0 +1,45 @@ +class WidgetConsumer < Deimos::ActiveRecordConsumer + record_class Widget + + # Optional override of the way to fetch records based on payload and + # key. Default is to use the key to search the primary key of the table. + # Only used in non-batch mode. + def fetch_record(klass, payload, key) + super + end + + # Optional override on how to set primary key for new records. + # Default is to set the class's primary key to the message's decoded key. + # Only used in non-batch mode. + def assign_key(record, payload, key) + super + end + + # Optional override of the default behavior, which is to call `destroy` + # on the record - e.g. you can replace this with "archiving" the record + # in some way. + # Only used in non-batch mode. + def destroy_record(record) + super + end + + # Optional override to change the attributes of the record before they + # are saved. + def record_attributes(payload, key) + super.merge(:some_field => 'some_value') + end + + # Optional override to change the attributes used for identifying records + def record_key(payload) + super + end + + # Optional override, returns true by default. + # When this method returns true, a record corresponding to the message + # is created/updated. + # When this method returns false, message processing is skipped and a + # corresponding record will NOT be created/updated. + def process_message?(payload) + super + end +end diff --git a/spec/snapshots/consumer_generator_migration.snap b/spec/snapshots/consumer_generator_migration.snap new file mode 100644 index 00000000..4689489e --- /dev/null +++ b/spec/snapshots/consumer_generator_migration.snap @@ -0,0 +1,22 @@ +class CreateWidgets < ActiveRecord::Migration[6.1] + def up + if table_exists?(:widgets) + warn "widgets already exists, exiting" + return + end + create_table :widgets do |t| + t.bigint :widget_id + t.string :name + + t.timestamps + + # TODO add indexes as necessary + end + end + + def down + return unless table_exists?(:widgets) + drop_table :widgets + end + +end diff --git a/spec/snapshots/consumer_generator_model.snap b/spec/snapshots/consumer_generator_model.snap new file mode 100644 index 00000000..a523af0a --- /dev/null +++ b/spec/snapshots/consumer_generator_model.snap @@ -0,0 +1,2 @@ +class Widget < ApplicationRecord +end diff --git a/spec/snapshots/consumer_generator_schema_classes.snap b/spec/snapshots/consumer_generator_schema_classes.snap new file mode 100644 index 00000000..b85db5e0 --- /dev/null +++ b/spec/snapshots/consumer_generator_schema_classes.snap @@ -0,0 +1,1379 @@ +spec/app/lib/schema_classes/my_namespace/generated.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module MyNamespace + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.Generated + class Generated < Deimos::SchemaClass::Record + + ### Secondary Schema Classes ### + # Autogenerated Schema for Record at com.my-namespace.ARecord + class ARecord < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [String] + attr_accessor :a_record_field + + # @override + def initialize(a_record_field: nil) + super + self.a_record_field = a_record_field + end + + # @override + def schema + 'ARecord' + end + + # @override + def namespace + 'com.my-namespace' + end + + # @override + def as_json(_opts={}) + { + 'a_record_field' => @a_record_field + } + end + end + + # Autogenerated Schema for Enum at com.my-namespace.AnEnum + class AnEnum < Deimos::SchemaClass::Enum + # @return ['sym1', 'sym2'] + attr_accessor :an_enum + + # @override + def symbols + %w(sym1 sym2) + end + end + + + ### Attribute Readers ### + # @return [AnEnum] + attr_reader :an_enum + # @return [ARecord] + attr_reader :a_record + + ### Attribute Accessors ### + # @return [String] + attr_accessor :a_string + # @return [Integer] + attr_accessor :a_int + # @return [Integer] + attr_accessor :a_long + # @return [Float] + attr_accessor :a_float + # @return [Float] + attr_accessor :a_double + # @return [nil, Integer] + attr_accessor :an_optional_int + # @return [Array] + attr_accessor :an_array + # @return [Hash] + attr_accessor :a_map + # @return [String] + attr_accessor :timestamp + # @return [String] + attr_accessor :message_id + + ### Attribute Writers ### + # @return [AnEnum] + def an_enum=(value) + @an_enum = AnEnum.initialize_from_value(value) + end + + # @return [ARecord] + def a_record=(value) + @a_record = ARecord.initialize_from_value(value) + end + + # @override + def initialize(a_string: nil, + a_int: nil, + a_long: nil, + a_float: nil, + a_double: nil, + an_optional_int: nil, + an_enum: nil, + an_array: nil, + a_map: nil, + timestamp: nil, + message_id: nil, + a_record: nil) + super + self.a_string = a_string + self.a_int = a_int + self.a_long = a_long + self.a_float = a_float + self.a_double = a_double + self.an_optional_int = an_optional_int + self.an_enum = an_enum + self.an_array = an_array + self.a_map = a_map + self.timestamp = timestamp + self.message_id = message_id + self.a_record = a_record + end + + # @override + def schema + 'Generated' + end + + # @override + def namespace + 'com.my-namespace' + end + + # @override + def as_json(_opts={}) + { + 'a_string' => @a_string, + 'a_int' => @a_int, + 'a_long' => @a_long, + 'a_float' => @a_float, + 'a_double' => @a_double, + 'an_optional_int' => @an_optional_int, + 'an_enum' => @an_enum&.as_json, + 'an_array' => @an_array, + 'a_map' => @a_map, + 'timestamp' => @timestamp, + 'message_id' => @message_id, + 'a_record' => @a_record&.as_json + } + end + end +end; end + + +spec/app/lib/schema_classes/my_namespace/my_nested_schema.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module MyNamespace + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.MyNestedSchema + class MyNestedSchema < Deimos::SchemaClass::Record + + ### Secondary Schema Classes ### + # Autogenerated Schema for Record at com.my-namespace.MyNestedRecord + class MyNestedRecord < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [Integer] + attr_accessor :some_int + # @return [Float] + attr_accessor :some_float + # @return [String] + attr_accessor :some_string + # @return [nil, Integer] + attr_accessor :some_optional_int + + # @override + def initialize(some_int: nil, + some_float: nil, + some_string: nil, + some_optional_int: nil) + super + self.some_int = some_int + self.some_float = some_float + self.some_string = some_string + self.some_optional_int = some_optional_int + end + + # @override + def schema + 'MyNestedRecord' + end + + # @override + def namespace + 'com.my-namespace' + end + + # @override + def as_json(_opts={}) + { + 'some_int' => @some_int, + 'some_float' => @some_float, + 'some_string' => @some_string, + 'some_optional_int' => @some_optional_int + } + end + end + + + ### Attribute Readers ### + # @return [MyNestedRecord] + attr_reader :some_nested_record + # @return [nil, MyNestedRecord] + attr_reader :some_optional_record + + ### Attribute Accessors ### + # @return [String] + attr_accessor :test_id + # @return [Float] + attr_accessor :test_float + # @return [Array] + attr_accessor :test_array + + ### Attribute Writers ### + # @return [MyNestedRecord] + def some_nested_record=(value) + @some_nested_record = MyNestedRecord.initialize_from_value(value) + end + + # @return [nil, MyNestedRecord] + def some_optional_record=(value) + @some_optional_record = MyNestedRecord.initialize_from_value(value) + end + + # @override + def initialize(test_id: nil, + test_float: nil, + test_array: nil, + some_nested_record: nil, + some_optional_record: nil) + super + self.test_id = test_id + self.test_float = test_float + self.test_array = test_array + self.some_nested_record = some_nested_record + self.some_optional_record = some_optional_record + end + + # @override + def schema + 'MyNestedSchema' + end + + # @override + def namespace + 'com.my-namespace' + end + + # @override + def as_json(_opts={}) + { + 'test_id' => @test_id, + 'test_float' => @test_float, + 'test_array' => @test_array, + 'some_nested_record' => @some_nested_record&.as_json, + 'some_optional_record' => @some_optional_record&.as_json + } + end + end +end; end + + +spec/app/lib/schema_classes/my_namespace/my_schema.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module MyNamespace + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.MySchema + class MySchema < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [String] + attr_accessor :test_id + # @return [Integer] + attr_accessor :some_int + + # @override + def initialize(test_id: nil, + some_int: nil) + super + self.test_id = test_id + self.some_int = some_int + end + + # @override + def schema + 'MySchema' + end + + # @override + def namespace + 'com.my-namespace' + end + + # @override + def as_json(_opts={}) + { + 'test_id' => @test_id, + 'some_int' => @some_int + } + end + end +end; end + + +spec/app/lib/schema_classes/my_namespace/my_schema_compound_key.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module MyNamespace + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.MySchemaCompound_key + class MySchemaCompoundKey < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [String] + attr_accessor :part_one + # @return [String] + attr_accessor :part_two + + # @override + def initialize(part_one: nil, + part_two: nil) + super + self.part_one = part_one + self.part_two = part_two + end + + # @override + def schema + 'MySchemaCompound_key' + end + + # @override + def namespace + 'com.my-namespace' + end + + # @override + def as_json(_opts={}) + { + 'part_one' => @part_one, + 'part_two' => @part_two + } + end + end +end; end + + +spec/app/lib/schema_classes/my_namespace/my_schema_id_key.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module MyNamespace + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.MySchemaId_key + class MySchemaIdKey < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [Integer] + attr_accessor :id + + # @override + def initialize(id: nil) + super + self.id = id + end + + # @override + def schema + 'MySchemaId_key' + end + + # @override + def namespace + 'com.my-namespace' + end + + # @override + def as_json(_opts={}) + { + 'id' => @id + } + end + end +end; end + + +spec/app/lib/schema_classes/my_namespace/my_schema_key.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module MyNamespace + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.MySchema_key + class MySchemaKey < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [String] + attr_accessor :test_id + + # @override + def initialize(test_id: nil) + super + self.test_id = test_id + end + + # @override + def schema + 'MySchema_key' + end + + # @override + def namespace + 'com.my-namespace' + end + + # @override + def as_json(_opts={}) + { + 'test_id' => @test_id + } + end + end +end; end + + +spec/app/lib/schema_classes/my_namespace/my_schema_with_boolean.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module MyNamespace + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.MySchemaWithBooleans + class MySchemaWithBoolean < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [String] + attr_accessor :test_id + # @return [Boolean] + attr_accessor :some_bool + + # @override + def initialize(test_id: nil, + some_bool: nil) + super + self.test_id = test_id + self.some_bool = some_bool + end + + # @override + def schema + 'MySchemaWithBooleans' + end + + # @override + def namespace + 'com.my-namespace' + end + + # @override + def as_json(_opts={}) + { + 'test_id' => @test_id, + 'some_bool' => @some_bool + } + end + end +end; end + + +spec/app/lib/schema_classes/my_namespace/my_schema_with_circular_reference.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module MyNamespace + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.MySchemaWithCircularReference + class MySchemaWithCircularReference < Deimos::SchemaClass::Record + + ### Secondary Schema Classes ### + # Autogenerated Schema for Record at com.my-namespace.Property + class Property < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [Boolean, Integer, Integer, Float, Float, String, Array, Hash] + attr_accessor :property + + # @override + def initialize(property: nil) + super + self.property = property + end + + # @override + def schema + 'Property' + end + + # @override + def namespace + 'com.my-namespace' + end + + # @override + def as_json(_opts={}) + { + 'property' => @property + } + end + end + + + ### Attribute Readers ### + # @return [Hash] + attr_reader :properties + + ### Attribute Writers ### + # @return [Hash] + def properties=(values) + @properties = values&.transform_values do |value| + Property.initialize_from_value(value) + end + end + + # @override + def initialize(properties: {}) + super + self.properties = properties + end + + # @override + def schema + 'MySchemaWithCircularReference' + end + + # @override + def namespace + 'com.my-namespace' + end + + # @override + def as_json(_opts={}) + { + 'properties' => @properties.transform_values { |v| v&.as_json } + } + end + end +end; end + + +spec/app/lib/schema_classes/my_namespace/my_schema_with_complex_type.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module MyNamespace + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.MySchemaWithComplexTypes + class MySchemaWithComplexType < Deimos::SchemaClass::Record + + ### Secondary Schema Classes ### + # Autogenerated Schema for Record at com.my-namespace.ARecord + class ARecord < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [String] + attr_accessor :a_record_field + + # @override + def initialize(a_record_field: nil) + super + self.a_record_field = a_record_field + end + + # @override + def schema + 'ARecord' + end + + # @override + def namespace + 'com.my-namespace' + end + + # @override + def as_json(_opts={}) + { + 'a_record_field' => @a_record_field + } + end + end + + # Autogenerated Schema for Enum at com.my-namespace.AnEnum + class AnEnum < Deimos::SchemaClass::Enum + # @return ['sym1', 'sym2'] + attr_accessor :an_enum + + # @override + def symbols + %w(sym1 sym2) + end + end + + # Autogenerated Schema for Enum at com.my-namespace.AnotherEnum + class AnotherEnum < Deimos::SchemaClass::Enum + # @return ['sym3', 'sym4'] + attr_accessor :another_enum + + # @override + def symbols + %w(sym3 sym4) + end + end + + # Autogenerated Schema for Enum at com.my-namespace.YetAnotherEnum + class YetAnotherEnum < Deimos::SchemaClass::Enum + # @return ['sym5', 'sym6'] + attr_accessor :yet_another_enum + + # @override + def symbols + %w(sym5 sym6) + end + end + + + ### Attribute Readers ### + # @return [ARecord] + attr_reader :some_record + # @return [nil, ARecord] + attr_reader :some_optional_record + # @return [Array] + attr_reader :some_record_array + # @return [Hash] + attr_reader :some_record_map + # @return [Array] + attr_reader :some_enum_array + # @return [nil, AnotherEnum] + attr_reader :some_optional_enum + # @return [YetAnotherEnum] + attr_reader :some_enum_with_default + + ### Attribute Accessors ### + # @return [String] + attr_accessor :test_id + # @return [Float] + attr_accessor :test_float + # @return [Array] + attr_accessor :test_string_array + # @return [Array] + attr_accessor :test_int_array + # @return [Integer, nil] + attr_accessor :test_optional_int + # @return [Hash] + attr_accessor :some_integer_map + + ### Attribute Writers ### + # @return [ARecord] + def some_record=(value) + @some_record = ARecord.initialize_from_value(value) + end + + # @return [nil, ARecord] + def some_optional_record=(value) + @some_optional_record = ARecord.initialize_from_value(value) + end + + # @return [Array] + def some_record_array=(values) + @some_record_array = values&.map do |value| + ARecord.initialize_from_value(value) + end + end + + # @return [Hash] + def some_record_map=(values) + @some_record_map = values&.transform_values do |value| + ARecord.initialize_from_value(value) + end + end + + # @return [Array] + def some_enum_array=(values) + @some_enum_array = values&.map do |value| + AnEnum.initialize_from_value(value) + end + end + + # @return [nil, AnotherEnum] + def some_optional_enum=(value) + @some_optional_enum = AnotherEnum.initialize_from_value(value) + end + + # @return [YetAnotherEnum] + def some_enum_with_default=(value) + @some_enum_with_default = YetAnotherEnum.initialize_from_value(value) + end + + # @override + def initialize(test_id: nil, + test_float: nil, + test_string_array: ["test"], + test_int_array: [123], + test_optional_int: 123, + some_integer_map: {"abc"=>123}, + some_record: {"a_record_field"=>"Test String"}, + some_optional_record: nil, + some_record_array: nil, + some_record_map: nil, + some_enum_array: nil, + some_optional_enum: nil, + some_enum_with_default: "sym6") + super + self.test_id = test_id + self.test_float = test_float + self.test_string_array = test_string_array + self.test_int_array = test_int_array + self.test_optional_int = test_optional_int + self.some_integer_map = some_integer_map + self.some_record = some_record + self.some_optional_record = some_optional_record + self.some_record_array = some_record_array + self.some_record_map = some_record_map + self.some_enum_array = some_enum_array + self.some_optional_enum = some_optional_enum + self.some_enum_with_default = some_enum_with_default + end + + # @override + def schema + 'MySchemaWithComplexTypes' + end + + # @override + def namespace + 'com.my-namespace' + end + + # @override + def as_json(_opts={}) + { + 'test_id' => @test_id, + 'test_float' => @test_float, + 'test_string_array' => @test_string_array, + 'test_int_array' => @test_int_array, + 'test_optional_int' => @test_optional_int, + 'some_integer_map' => @some_integer_map, + 'some_record' => @some_record&.as_json, + 'some_optional_record' => @some_optional_record&.as_json, + 'some_record_array' => @some_record_array.map { |v| v&.as_json }, + 'some_record_map' => @some_record_map.transform_values { |v| v&.as_json }, + 'some_enum_array' => @some_enum_array.map { |v| v&.as_json }, + 'some_optional_enum' => @some_optional_enum&.as_json, + 'some_enum_with_default' => @some_enum_with_default&.as_json + } + end + end +end; end + + +spec/app/lib/schema_classes/my_namespace/my_schema_with_date_time.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module MyNamespace + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.MySchemaWithDateTimes + class MySchemaWithDateTime < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [String] + attr_accessor :test_id + # @return [Integer, nil] + attr_accessor :updated_at + # @return [nil, Integer] + attr_accessor :some_int + # @return [nil, Integer] + attr_accessor :some_datetime_int + # @return [String] + attr_accessor :timestamp + + # @override + def initialize(test_id: nil, + updated_at: nil, + some_int: nil, + some_datetime_int: nil, + timestamp: nil) + super + self.test_id = test_id + self.updated_at = updated_at + self.some_int = some_int + self.some_datetime_int = some_datetime_int + self.timestamp = timestamp + end + + # @override + def schema + 'MySchemaWithDateTimes' + end + + # @override + def namespace + 'com.my-namespace' + end + + # @override + def as_json(_opts={}) + { + 'test_id' => @test_id, + 'updated_at' => @updated_at, + 'some_int' => @some_int, + 'some_datetime_int' => @some_datetime_int, + 'timestamp' => @timestamp + } + end + end +end; end + + +spec/app/lib/schema_classes/my_namespace/my_schema_with_id.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module MyNamespace + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.MySchemaWithId + class MySchemaWithId < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [String] + attr_accessor :test_id + # @return [Integer] + attr_accessor :some_int + # @return [String] + attr_accessor :message_id + # @return [String] + attr_accessor :timestamp + + # @override + def initialize(test_id: nil, + some_int: nil, + message_id: nil, + timestamp: nil) + super + self.test_id = test_id + self.some_int = some_int + self.message_id = message_id + self.timestamp = timestamp + end + + # @override + def schema + 'MySchemaWithId' + end + + # @override + def namespace + 'com.my-namespace' + end + + # @override + def as_json(_opts={}) + { + 'test_id' => @test_id, + 'some_int' => @some_int, + 'message_id' => @message_id, + 'timestamp' => @timestamp + } + end + end +end; end + + +spec/app/lib/schema_classes/my_namespace/my_schema_with_unique_id.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module MyNamespace + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.MySchemaWithUniqueId + class MySchemaWithUniqueId < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [Integer] + attr_accessor :id + # @return [String] + attr_accessor :test_id + # @return [Integer] + attr_accessor :some_int + # @return [String] + attr_accessor :message_id + # @return [String] + attr_accessor :timestamp + + # @override + def initialize(id: nil, + test_id: nil, + some_int: nil, + message_id: nil, + timestamp: nil) + super + self.id = id + self.test_id = test_id + self.some_int = some_int + self.message_id = message_id + self.timestamp = timestamp + end + + # @override + def schema + 'MySchemaWithUniqueId' + end + + # @override + def namespace + 'com.my-namespace' + end + + # @override + def as_json(_opts={}) + { + 'id' => @id, + 'test_id' => @test_id, + 'some_int' => @some_int, + 'message_id' => @message_id, + 'timestamp' => @timestamp + } + end + end +end; end + + +spec/app/lib/schema_classes/my_namespace/wibble.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module MyNamespace + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.Wibble + class Wibble < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [Integer] + attr_accessor :id + # @return [Integer] + attr_accessor :wibble_id + # @return [String] + attr_accessor :name + # @return [String] + attr_accessor :floop + # @return [Integer] + attr_accessor :birthday_int + # @return [Integer] + attr_accessor :birthday_long + # @return [nil, Integer] + attr_accessor :birthday_optional + # @return [Integer] + attr_accessor :updated_at + # @return [Integer] + attr_accessor :created_at + + # @override + def initialize(id: nil, + wibble_id: nil, + name: nil, + floop: nil, + birthday_int: nil, + birthday_long: nil, + birthday_optional: nil, + updated_at: nil, + created_at: nil) + super + self.id = id + self.wibble_id = wibble_id + self.name = name + self.floop = floop + self.birthday_int = birthday_int + self.birthday_long = birthday_long + self.birthday_optional = birthday_optional + self.updated_at = updated_at + self.created_at = created_at + end + + # @override + def schema + 'Wibble' + end + + # @override + def namespace + 'com.my-namespace' + end + + # @override + def as_json(_opts={}) + { + 'id' => @id, + 'wibble_id' => @wibble_id, + 'name' => @name, + 'floop' => @floop, + 'birthday_int' => @birthday_int, + 'birthday_long' => @birthday_long, + 'birthday_optional' => @birthday_optional, + 'updated_at' => @updated_at, + 'created_at' => @created_at + } + end + end +end; end + + +spec/app/lib/schema_classes/my_namespace/widget.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module MyNamespace + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.Widget + class Widget < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [Integer] + attr_accessor :id + # @return [Integer] + attr_accessor :widget_id + # @return [String] + attr_accessor :name + # @return [Integer] + attr_accessor :updated_at + # @return [Integer] + attr_accessor :created_at + + # @override + def initialize(id: nil, + widget_id: nil, + name: nil, + updated_at: nil, + created_at: nil) + super + self.id = id + self.widget_id = widget_id + self.name = name + self.updated_at = updated_at + self.created_at = created_at + end + + # @override + def schema + 'Widget' + end + + # @override + def namespace + 'com.my-namespace' + end + + def self.tombstone(key) + record = self.allocate + record.tombstone_key = key + record.a_string = key + record + end + + # @override + def as_json(_opts={}) + { + 'id' => @id, + 'widget_id' => @widget_id, + 'name' => @name, + 'updated_at' => @updated_at, + 'created_at' => @created_at + } + end + end +end; end + + +spec/app/lib/schema_classes/my_namespace/widget_the_second.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module MyNamespace + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.WidgetTheSecond + class WidgetTheSecond < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [Integer] + attr_accessor :id + # @return [Integer] + attr_accessor :widget_id + # @return [String] + attr_accessor :model_id + # @return [Integer] + attr_accessor :updated_at + # @return [Integer] + attr_accessor :created_at + + # @override + def initialize(id: nil, + widget_id: nil, + model_id: nil, + updated_at: nil, + created_at: nil) + super + self.id = id + self.widget_id = widget_id + self.model_id = model_id + self.updated_at = updated_at + self.created_at = created_at + end + + # @override + def schema + 'WidgetTheSecond' + end + + # @override + def namespace + 'com.my-namespace' + end + + # @override + def as_json(_opts={}) + { + 'id' => @id, + 'widget_id' => @widget_id, + 'model_id' => @model_id, + 'updated_at' => @updated_at, + 'created_at' => @created_at + } + end + end +end; end + + +spec/app/lib/schema_classes/request/create_topic.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module Request + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.request.CreateTopic + class CreateTopic < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [String] + attr_accessor :request_id + + # @override + def initialize(request_id: nil) + super + self.request_id = request_id + end + + # @override + def schema + 'CreateTopic' + end + + # @override + def namespace + 'com.my-namespace.request' + end + + # @override + def as_json(_opts={}) + { + 'request_id' => @request_id + } + end + end +end; end + + +spec/app/lib/schema_classes/request/index.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module Request + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.request.Index + class Index < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [String] + attr_accessor :request_id + + # @override + def initialize(request_id: nil) + super + self.request_id = request_id + end + + # @override + def schema + 'Index' + end + + # @override + def namespace + 'com.my-namespace.request' + end + + # @override + def as_json(_opts={}) + { + 'request_id' => @request_id + } + end + end +end; end + + +spec/app/lib/schema_classes/request/update_request.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module Request + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.request.UpdateRequest + class UpdateRequest < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [String] + attr_accessor :update_request_id + + # @override + def initialize(update_request_id: nil) + super + self.update_request_id = update_request_id + end + + # @override + def schema + 'UpdateRequest' + end + + # @override + def namespace + 'com.my-namespace.request' + end + + # @override + def as_json(_opts={}) + { + 'update_request_id' => @update_request_id + } + end + end +end; end + + +spec/app/lib/schema_classes/response/create_topic.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module Response + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.response.CreateTopic + class CreateTopic < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [String] + attr_accessor :response_id + + # @override + def initialize(response_id: nil) + super + self.response_id = response_id + end + + # @override + def schema + 'CreateTopic' + end + + # @override + def namespace + 'com.my-namespace.response' + end + + # @override + def as_json(_opts={}) + { + 'response_id' => @response_id + } + end + end +end; end + + +spec/app/lib/schema_classes/response/index.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module Response + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.response.Index + class Index < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [String] + attr_accessor :response_id + + # @override + def initialize(response_id: nil) + super + self.response_id = response_id + end + + # @override + def schema + 'Index' + end + + # @override + def namespace + 'com.my-namespace.response' + end + + # @override + def as_json(_opts={}) + { + 'response_id' => @response_id + } + end + end +end; end + + +spec/app/lib/schema_classes/response/update_response.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas; module Response + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.response.UpdateResponse + class UpdateResponse < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [String] + attr_accessor :update_response_id + + # @override + def initialize(update_response_id: nil) + super + self.update_response_id = update_response_id + end + + # @override + def schema + 'UpdateResponse' + end + + # @override + def namespace + 'com.my-namespace.response' + end + + # @override + def as_json(_opts={}) + { + 'update_response_id' => @update_response_id + } + end + end +end; end + From e07ae14e27ab1c6e0397bb26d16772c265832228 Mon Sep 17 00:00:00 2001 From: jenchanflipp Date: Wed, 16 Nov 2022 10:29:10 -0500 Subject: [PATCH 2/7] GUILD-690: Initial commit of ActiveRecordConsumerGenerator code --- CHANGELOG.md | 7 ++ .../active_record_consumer_generator_spec.rb | 93 +++++++++++++++++++ 2 files changed, 100 insertions(+) create mode 100644 spec/generators/active_record_consumer_generator_spec.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index c82eb648..51478758 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## UNRELEASED +# 1.18.3 - 2022-11-16 + +### Features :star: + +- Added ActiveRecordConsumerGenerator, which streamlines the process into a single flow +- Creates a database migration, Rails model, consumer, consumer config, and Deimos schema classes + # 1.18.2 - 2022-11-14 - Fixes bug related to wait between runs when no records are fetched by updating poll_info diff --git a/spec/generators/active_record_consumer_generator_spec.rb b/spec/generators/active_record_consumer_generator_spec.rb new file mode 100644 index 00000000..09190db9 --- /dev/null +++ b/spec/generators/active_record_consumer_generator_spec.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +require 'generators/deimos/active_record_consumer_generator' +require 'fileutils' +require 'tempfile' + +RSpec.describe Deimos::Generators::ActiveRecordConsumerGenerator do + let(:db_migration_path) { 'db/migrate' } + let(:model_path) { 'app/models' } + let(:consumer_path) { 'app/lib/kafka/models' } + let(:config_path) { 'config/initializers' } + let(:schema_class_path) { 'spec/app/lib/schema_classes' } + + after(:each) do + FileUtils.rm_rf('db') if File.exist?('db') + FileUtils.rm_rf('app') if File.exist?('app') + FileUtils.rm_rf('config') if File.exist?('config') + end + + before(:each) do + Deimos.config.reset! + Deimos.configure do + schema.path('spec/schemas/') + schema.generated_class_path('spec/app/lib/schema_classes') + schema.backend(:avro_local) + schema.generate_namespace_folders(true) + end + end + + it 'should generate a migration, model, consumer class, config and schema classes' do + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end + end + + expect(Dir["#{db_migration_path}/*.rb"]).to be_empty + expect(Dir["#{model_path}/*.rb"]).to be_empty + expect(Dir["#{config_path}/*.rb"]).to be_empty + expect(Dir["#{schema_class_path}/*.rb"]).to be_empty + + described_class.start(['com.my-namespace.Widget']) + + files = Dir["#{db_migration_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_migration') + + files = Dir["#{model_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_model') + + files = Dir["#{consumer_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_consumer_class') + + files = Dir["#{config_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_config') + + files = Dir["#{schema_class_path}/**/*.rb"].map { |f| [f, File.read(f)] }.to_h + expect(files).to match_snapshot('consumer_generator_schema_classes', snapshot_serializer: MultiFileSerializer) + end + + it 'should generate a migration, model, consumer class, schema classes and edit the passed in config file' do + config_file_name = 'my_config.config' + FileUtils.mkdir_p(config_path) + Tempfile.new(config_file_name, config_path) + config_path_argument = "#{config_path}/#{config_file_name}" + + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end + end + + expect(Dir["#{db_migration_path}/*.rb"]).to be_empty + expect(Dir["#{model_path}/*.rb"]).to be_empty + expect(Dir["#{schema_class_path}/*.rb"]).to be_empty + + described_class.start(['com.my-namespace.Widget', config_path_argument]) + + expect(File.read(config_path_argument)).to match_snapshot('consumer_generator_config_with_config_arg') + end + +end From 8b3130de421d0ed5d354079bff10fcc92d57c603 Mon Sep 17 00:00:00 2001 From: jenchanflipp <72824964+jenchanflipp@users.noreply.github.com> Date: Tue, 24 Jan 2023 15:08:07 -0500 Subject: [PATCH 3/7] GUILD-690: Invoke active record generator and schema class generator with new option to skip generating classes for all schema files --- .../active_record_consumer_generator.rb | 80 +- .../deimos/schema_class_generator.rb | 4 +- .../active_record_consumer_generator_spec.rb | 27 +- .../generators/schema_class_generator_spec.rb | 21 + .../consumer_generator_schema_classes.snap | 1316 ----------------- ...idget-skip-generate-from-schema-files.snap | 65 + 6 files changed, 159 insertions(+), 1354 deletions(-) create mode 100644 spec/snapshots/widget-skip-generate-from-schema-files.snap diff --git a/lib/generators/deimos/active_record_consumer_generator.rb b/lib/generators/deimos/active_record_consumer_generator.rb index f0d62b20..238906d3 100644 --- a/lib/generators/deimos/active_record_consumer_generator.rb +++ b/lib/generators/deimos/active_record_consumer_generator.rb @@ -3,6 +3,7 @@ require 'rails/generators' require 'rails/generators/active_record/migration' require 'generators/deimos/schema_class_generator' +require 'generators/deimos/active_record_generator' require 'rails/version' require 'erb' @@ -20,21 +21,47 @@ class ActiveRecordConsumerGenerator < Rails::Generators::Base else include ActiveRecord::Generators::Migration end + + # @return [Array] + KEY_CONFIG_OPTIONS = %w(none plain schema field).freeze + + # @return [Array] + KEY_CONFIG_OPTIONS_BOOL = %w(none plain).freeze + + # @return [Array] + KEY_CONFIG_OPTIONS_STRING = %w(schema field).freeze + source_root File.expand_path('active_record_consumer/templates', __dir__) argument :full_schema, desc: 'The fully qualified schema name.', required: true + argument :key_config_type, desc: 'The kafka message key configuration type.', required: true + argument :key_config_value, desc: 'The kafka message key configuration value.', required: true argument :config_path, desc: 'The path to the deimos configuration file, relative to the root directory.', required: false no_commands do - # Creates database migration for creating new table - def create_db_migration - migration_template('migration.rb', "#{db_migrate_path}/create_#{table_name.underscore}.rb") + def validate_arguments + # key_config_hash = {} + # colon = self.full_schema.rindex(':') + # value = key_config[colon+1...-1].strip + # if value.rindex("'").present? + # key_config_hash[key_config[0...colon]] = key_config[colon+1...-1] + # + # self.full_schema[0...last_dot] + + @key_config_hash = {} + if KEY_CONFIG_OPTIONS_BOOL.include?(key_config_type) + @key_config_hash[key_config_type] = true + elsif KEY_CONFIG_OPTIONS_STRING.include?(key_config_type) + @key_config_hash[key_config_type] = key_config_value + else + return ' nil' + end end - # Creates Rails Model - def create_rails_model - template('model.rb', "app/models/#{model_name}.rb") + # Creates database migration for creating new table and Rails Model + def create_db_migration_rails_model + Deimos::Generators::ActiveRecordGenerator.start([table_name,full_schema]) end # Creates Kafka Consumer file @@ -59,29 +86,7 @@ def create_consumer_config # Generates schema classes def create_deimos_schema_class - Deimos::Generators::SchemaClassGenerator.start - end - - # @return [String] - def db_migrate_path - if defined?(Rails.application) && Rails.application - paths = Rails.application.config.paths['db/migrate'] - paths.respond_to?(:to_ary) ? paths.to_ary.first : paths.to_a.first - else - 'db/migrate' - end - end - - # @return [String] - def migration_version - "[#{ActiveRecord::Migration.current_version}]" - rescue StandardError - '' - end - - # @return [String] - def table_class - table_name.classify + Deimos::Generators::SchemaClassGenerator.start(['--skip_generate_from_schema_files']) end # @return [String] @@ -140,12 +145,25 @@ def key_field # desc 'Generate necessary files and configuration for a new Active Record Consumer.' # @return [void] def generate - create_db_migration - create_rails_model + # if yes?('Would you like to install Rspec?') + # gem 'rspec-rails', group: :test + # after_bundle { generate 'rspec:install' } + # end + validate_arguments + create_db_migration_rails_model create_consumer create_consumer_config create_deimos_schema_class end + + private + + # Determines if Schema Class Generation can be run. + # @raise if Schema Backend is not of a Avro-based class + def _validate + backend = Deimos.config.schema.backend.to_s + raise 'Schema Class Generation requires an Avro-based Schema Backend' if backend !~ /^avro/ + end end end end diff --git a/lib/generators/deimos/schema_class_generator.rb b/lib/generators/deimos/schema_class_generator.rb index 0e30541e..bc3891a8 100644 --- a/lib/generators/deimos/schema_class_generator.rb +++ b/lib/generators/deimos/schema_class_generator.rb @@ -26,6 +26,8 @@ class SchemaClassGenerator < Rails::Generators::Base source_root File.expand_path('schema_class/templates', __dir__) + class_option :skip_generate_from_schema_files, type: :boolean, default: false + no_commands do # Retrieve the fields from this Avro Schema # @param schema [Avro::Schema::NamedSchema] @@ -175,7 +177,7 @@ def generate generate_classes(schema_name, namespace, config.key_config) end - generate_from_schema_files(found_schemas) + generate_from_schema_files(found_schemas) unless options[:skip_generate_from_schema_files] end diff --git a/spec/generators/active_record_consumer_generator_spec.rb b/spec/generators/active_record_consumer_generator_spec.rb index 09190db9..ea67339b 100644 --- a/spec/generators/active_record_consumer_generator_spec.rb +++ b/spec/generators/active_record_consumer_generator_spec.rb @@ -27,7 +27,7 @@ end end - it 'should generate a migration, model, consumer class, config and schema classes' do + it 'should generate a migration, model, consumer class, config and schema class' do Deimos.configure do consumer do class_name 'ConsumerTest::MyConsumer' @@ -43,7 +43,7 @@ expect(Dir["#{config_path}/*.rb"]).to be_empty expect(Dir["#{schema_class_path}/*.rb"]).to be_empty - described_class.start(['com.my-namespace.Widget']) + described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key']) files = Dir["#{db_migration_path}/*.rb"] expect(files.length).to eq(1) @@ -61,11 +61,11 @@ expect(files.length).to eq(1) expect(File.read(files[0])).to match_snapshot('consumer_generator_config') - files = Dir["#{schema_class_path}/**/*.rb"].map { |f| [f, File.read(f)] }.to_h - expect(files).to match_snapshot('consumer_generator_schema_classes', snapshot_serializer: MultiFileSerializer) + files = Dir["#{schema_class_path}/*/*.rb"] + expect(File.read(files[0])).to match_snapshot('consumer_generator_schema_classes') end - it 'should generate a migration, model, consumer class, schema classes and edit the passed in config file' do + it 'should generate a migration, model, consumer class, schema class and edit the passed in config file' do config_file_name = 'my_config.config' FileUtils.mkdir_p(config_path) Tempfile.new(config_file_name, config_path) @@ -85,9 +85,24 @@ expect(Dir["#{model_path}/*.rb"]).to be_empty expect(Dir["#{schema_class_path}/*.rb"]).to be_empty - described_class.start(['com.my-namespace.Widget', config_path_argument]) + described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key',config_path_argument]) + + files = Dir["#{db_migration_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_migration') + + files = Dir["#{model_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_model') + + files = Dir["#{consumer_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_consumer_class') expect(File.read(config_path_argument)).to match_snapshot('consumer_generator_config_with_config_arg') + + files = Dir["#{schema_class_path}/*/*.rb"] + expect(File.read(files[0])).to match_snapshot('consumer_generator_schema_classes') end end diff --git a/spec/generators/schema_class_generator_spec.rb b/spec/generators/schema_class_generator_spec.rb index 00c87437..904f0140 100644 --- a/spec/generators/schema_class_generator_spec.rb +++ b/spec/generators/schema_class_generator_spec.rb @@ -255,4 +255,25 @@ def dump(value) end end + context 'with skip_generate_from_schema_files option' do + before(:each) do + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end + end + end + + it 'should only generate class for consumer defined in config' do + Deimos.with_config('schema.nest_child_schemas' => false) do + described_class.start(['--skip_generate_from_schema_files']) + expect(files).to match_snapshot('widget-skip-generate-from-schema-files', snapshot_serializer: MultiFileSerializer) + end + end + end + end diff --git a/spec/snapshots/consumer_generator_schema_classes.snap b/spec/snapshots/consumer_generator_schema_classes.snap index b85db5e0..614a0310 100644 --- a/spec/snapshots/consumer_generator_schema_classes.snap +++ b/spec/snapshots/consumer_generator_schema_classes.snap @@ -1,1025 +1,3 @@ -spec/app/lib/schema_classes/my_namespace/generated.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module MyNamespace - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.Generated - class Generated < Deimos::SchemaClass::Record - - ### Secondary Schema Classes ### - # Autogenerated Schema for Record at com.my-namespace.ARecord - class ARecord < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [String] - attr_accessor :a_record_field - - # @override - def initialize(a_record_field: nil) - super - self.a_record_field = a_record_field - end - - # @override - def schema - 'ARecord' - end - - # @override - def namespace - 'com.my-namespace' - end - - # @override - def as_json(_opts={}) - { - 'a_record_field' => @a_record_field - } - end - end - - # Autogenerated Schema for Enum at com.my-namespace.AnEnum - class AnEnum < Deimos::SchemaClass::Enum - # @return ['sym1', 'sym2'] - attr_accessor :an_enum - - # @override - def symbols - %w(sym1 sym2) - end - end - - - ### Attribute Readers ### - # @return [AnEnum] - attr_reader :an_enum - # @return [ARecord] - attr_reader :a_record - - ### Attribute Accessors ### - # @return [String] - attr_accessor :a_string - # @return [Integer] - attr_accessor :a_int - # @return [Integer] - attr_accessor :a_long - # @return [Float] - attr_accessor :a_float - # @return [Float] - attr_accessor :a_double - # @return [nil, Integer] - attr_accessor :an_optional_int - # @return [Array] - attr_accessor :an_array - # @return [Hash] - attr_accessor :a_map - # @return [String] - attr_accessor :timestamp - # @return [String] - attr_accessor :message_id - - ### Attribute Writers ### - # @return [AnEnum] - def an_enum=(value) - @an_enum = AnEnum.initialize_from_value(value) - end - - # @return [ARecord] - def a_record=(value) - @a_record = ARecord.initialize_from_value(value) - end - - # @override - def initialize(a_string: nil, - a_int: nil, - a_long: nil, - a_float: nil, - a_double: nil, - an_optional_int: nil, - an_enum: nil, - an_array: nil, - a_map: nil, - timestamp: nil, - message_id: nil, - a_record: nil) - super - self.a_string = a_string - self.a_int = a_int - self.a_long = a_long - self.a_float = a_float - self.a_double = a_double - self.an_optional_int = an_optional_int - self.an_enum = an_enum - self.an_array = an_array - self.a_map = a_map - self.timestamp = timestamp - self.message_id = message_id - self.a_record = a_record - end - - # @override - def schema - 'Generated' - end - - # @override - def namespace - 'com.my-namespace' - end - - # @override - def as_json(_opts={}) - { - 'a_string' => @a_string, - 'a_int' => @a_int, - 'a_long' => @a_long, - 'a_float' => @a_float, - 'a_double' => @a_double, - 'an_optional_int' => @an_optional_int, - 'an_enum' => @an_enum&.as_json, - 'an_array' => @an_array, - 'a_map' => @a_map, - 'timestamp' => @timestamp, - 'message_id' => @message_id, - 'a_record' => @a_record&.as_json - } - end - end -end; end - - -spec/app/lib/schema_classes/my_namespace/my_nested_schema.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module MyNamespace - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.MyNestedSchema - class MyNestedSchema < Deimos::SchemaClass::Record - - ### Secondary Schema Classes ### - # Autogenerated Schema for Record at com.my-namespace.MyNestedRecord - class MyNestedRecord < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [Integer] - attr_accessor :some_int - # @return [Float] - attr_accessor :some_float - # @return [String] - attr_accessor :some_string - # @return [nil, Integer] - attr_accessor :some_optional_int - - # @override - def initialize(some_int: nil, - some_float: nil, - some_string: nil, - some_optional_int: nil) - super - self.some_int = some_int - self.some_float = some_float - self.some_string = some_string - self.some_optional_int = some_optional_int - end - - # @override - def schema - 'MyNestedRecord' - end - - # @override - def namespace - 'com.my-namespace' - end - - # @override - def as_json(_opts={}) - { - 'some_int' => @some_int, - 'some_float' => @some_float, - 'some_string' => @some_string, - 'some_optional_int' => @some_optional_int - } - end - end - - - ### Attribute Readers ### - # @return [MyNestedRecord] - attr_reader :some_nested_record - # @return [nil, MyNestedRecord] - attr_reader :some_optional_record - - ### Attribute Accessors ### - # @return [String] - attr_accessor :test_id - # @return [Float] - attr_accessor :test_float - # @return [Array] - attr_accessor :test_array - - ### Attribute Writers ### - # @return [MyNestedRecord] - def some_nested_record=(value) - @some_nested_record = MyNestedRecord.initialize_from_value(value) - end - - # @return [nil, MyNestedRecord] - def some_optional_record=(value) - @some_optional_record = MyNestedRecord.initialize_from_value(value) - end - - # @override - def initialize(test_id: nil, - test_float: nil, - test_array: nil, - some_nested_record: nil, - some_optional_record: nil) - super - self.test_id = test_id - self.test_float = test_float - self.test_array = test_array - self.some_nested_record = some_nested_record - self.some_optional_record = some_optional_record - end - - # @override - def schema - 'MyNestedSchema' - end - - # @override - def namespace - 'com.my-namespace' - end - - # @override - def as_json(_opts={}) - { - 'test_id' => @test_id, - 'test_float' => @test_float, - 'test_array' => @test_array, - 'some_nested_record' => @some_nested_record&.as_json, - 'some_optional_record' => @some_optional_record&.as_json - } - end - end -end; end - - -spec/app/lib/schema_classes/my_namespace/my_schema.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module MyNamespace - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.MySchema - class MySchema < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [String] - attr_accessor :test_id - # @return [Integer] - attr_accessor :some_int - - # @override - def initialize(test_id: nil, - some_int: nil) - super - self.test_id = test_id - self.some_int = some_int - end - - # @override - def schema - 'MySchema' - end - - # @override - def namespace - 'com.my-namespace' - end - - # @override - def as_json(_opts={}) - { - 'test_id' => @test_id, - 'some_int' => @some_int - } - end - end -end; end - - -spec/app/lib/schema_classes/my_namespace/my_schema_compound_key.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module MyNamespace - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.MySchemaCompound_key - class MySchemaCompoundKey < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [String] - attr_accessor :part_one - # @return [String] - attr_accessor :part_two - - # @override - def initialize(part_one: nil, - part_two: nil) - super - self.part_one = part_one - self.part_two = part_two - end - - # @override - def schema - 'MySchemaCompound_key' - end - - # @override - def namespace - 'com.my-namespace' - end - - # @override - def as_json(_opts={}) - { - 'part_one' => @part_one, - 'part_two' => @part_two - } - end - end -end; end - - -spec/app/lib/schema_classes/my_namespace/my_schema_id_key.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module MyNamespace - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.MySchemaId_key - class MySchemaIdKey < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [Integer] - attr_accessor :id - - # @override - def initialize(id: nil) - super - self.id = id - end - - # @override - def schema - 'MySchemaId_key' - end - - # @override - def namespace - 'com.my-namespace' - end - - # @override - def as_json(_opts={}) - { - 'id' => @id - } - end - end -end; end - - -spec/app/lib/schema_classes/my_namespace/my_schema_key.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module MyNamespace - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.MySchema_key - class MySchemaKey < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [String] - attr_accessor :test_id - - # @override - def initialize(test_id: nil) - super - self.test_id = test_id - end - - # @override - def schema - 'MySchema_key' - end - - # @override - def namespace - 'com.my-namespace' - end - - # @override - def as_json(_opts={}) - { - 'test_id' => @test_id - } - end - end -end; end - - -spec/app/lib/schema_classes/my_namespace/my_schema_with_boolean.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module MyNamespace - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.MySchemaWithBooleans - class MySchemaWithBoolean < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [String] - attr_accessor :test_id - # @return [Boolean] - attr_accessor :some_bool - - # @override - def initialize(test_id: nil, - some_bool: nil) - super - self.test_id = test_id - self.some_bool = some_bool - end - - # @override - def schema - 'MySchemaWithBooleans' - end - - # @override - def namespace - 'com.my-namespace' - end - - # @override - def as_json(_opts={}) - { - 'test_id' => @test_id, - 'some_bool' => @some_bool - } - end - end -end; end - - -spec/app/lib/schema_classes/my_namespace/my_schema_with_circular_reference.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module MyNamespace - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.MySchemaWithCircularReference - class MySchemaWithCircularReference < Deimos::SchemaClass::Record - - ### Secondary Schema Classes ### - # Autogenerated Schema for Record at com.my-namespace.Property - class Property < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [Boolean, Integer, Integer, Float, Float, String, Array, Hash] - attr_accessor :property - - # @override - def initialize(property: nil) - super - self.property = property - end - - # @override - def schema - 'Property' - end - - # @override - def namespace - 'com.my-namespace' - end - - # @override - def as_json(_opts={}) - { - 'property' => @property - } - end - end - - - ### Attribute Readers ### - # @return [Hash] - attr_reader :properties - - ### Attribute Writers ### - # @return [Hash] - def properties=(values) - @properties = values&.transform_values do |value| - Property.initialize_from_value(value) - end - end - - # @override - def initialize(properties: {}) - super - self.properties = properties - end - - # @override - def schema - 'MySchemaWithCircularReference' - end - - # @override - def namespace - 'com.my-namespace' - end - - # @override - def as_json(_opts={}) - { - 'properties' => @properties.transform_values { |v| v&.as_json } - } - end - end -end; end - - -spec/app/lib/schema_classes/my_namespace/my_schema_with_complex_type.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module MyNamespace - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.MySchemaWithComplexTypes - class MySchemaWithComplexType < Deimos::SchemaClass::Record - - ### Secondary Schema Classes ### - # Autogenerated Schema for Record at com.my-namespace.ARecord - class ARecord < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [String] - attr_accessor :a_record_field - - # @override - def initialize(a_record_field: nil) - super - self.a_record_field = a_record_field - end - - # @override - def schema - 'ARecord' - end - - # @override - def namespace - 'com.my-namespace' - end - - # @override - def as_json(_opts={}) - { - 'a_record_field' => @a_record_field - } - end - end - - # Autogenerated Schema for Enum at com.my-namespace.AnEnum - class AnEnum < Deimos::SchemaClass::Enum - # @return ['sym1', 'sym2'] - attr_accessor :an_enum - - # @override - def symbols - %w(sym1 sym2) - end - end - - # Autogenerated Schema for Enum at com.my-namespace.AnotherEnum - class AnotherEnum < Deimos::SchemaClass::Enum - # @return ['sym3', 'sym4'] - attr_accessor :another_enum - - # @override - def symbols - %w(sym3 sym4) - end - end - - # Autogenerated Schema for Enum at com.my-namespace.YetAnotherEnum - class YetAnotherEnum < Deimos::SchemaClass::Enum - # @return ['sym5', 'sym6'] - attr_accessor :yet_another_enum - - # @override - def symbols - %w(sym5 sym6) - end - end - - - ### Attribute Readers ### - # @return [ARecord] - attr_reader :some_record - # @return [nil, ARecord] - attr_reader :some_optional_record - # @return [Array] - attr_reader :some_record_array - # @return [Hash] - attr_reader :some_record_map - # @return [Array] - attr_reader :some_enum_array - # @return [nil, AnotherEnum] - attr_reader :some_optional_enum - # @return [YetAnotherEnum] - attr_reader :some_enum_with_default - - ### Attribute Accessors ### - # @return [String] - attr_accessor :test_id - # @return [Float] - attr_accessor :test_float - # @return [Array] - attr_accessor :test_string_array - # @return [Array] - attr_accessor :test_int_array - # @return [Integer, nil] - attr_accessor :test_optional_int - # @return [Hash] - attr_accessor :some_integer_map - - ### Attribute Writers ### - # @return [ARecord] - def some_record=(value) - @some_record = ARecord.initialize_from_value(value) - end - - # @return [nil, ARecord] - def some_optional_record=(value) - @some_optional_record = ARecord.initialize_from_value(value) - end - - # @return [Array] - def some_record_array=(values) - @some_record_array = values&.map do |value| - ARecord.initialize_from_value(value) - end - end - - # @return [Hash] - def some_record_map=(values) - @some_record_map = values&.transform_values do |value| - ARecord.initialize_from_value(value) - end - end - - # @return [Array] - def some_enum_array=(values) - @some_enum_array = values&.map do |value| - AnEnum.initialize_from_value(value) - end - end - - # @return [nil, AnotherEnum] - def some_optional_enum=(value) - @some_optional_enum = AnotherEnum.initialize_from_value(value) - end - - # @return [YetAnotherEnum] - def some_enum_with_default=(value) - @some_enum_with_default = YetAnotherEnum.initialize_from_value(value) - end - - # @override - def initialize(test_id: nil, - test_float: nil, - test_string_array: ["test"], - test_int_array: [123], - test_optional_int: 123, - some_integer_map: {"abc"=>123}, - some_record: {"a_record_field"=>"Test String"}, - some_optional_record: nil, - some_record_array: nil, - some_record_map: nil, - some_enum_array: nil, - some_optional_enum: nil, - some_enum_with_default: "sym6") - super - self.test_id = test_id - self.test_float = test_float - self.test_string_array = test_string_array - self.test_int_array = test_int_array - self.test_optional_int = test_optional_int - self.some_integer_map = some_integer_map - self.some_record = some_record - self.some_optional_record = some_optional_record - self.some_record_array = some_record_array - self.some_record_map = some_record_map - self.some_enum_array = some_enum_array - self.some_optional_enum = some_optional_enum - self.some_enum_with_default = some_enum_with_default - end - - # @override - def schema - 'MySchemaWithComplexTypes' - end - - # @override - def namespace - 'com.my-namespace' - end - - # @override - def as_json(_opts={}) - { - 'test_id' => @test_id, - 'test_float' => @test_float, - 'test_string_array' => @test_string_array, - 'test_int_array' => @test_int_array, - 'test_optional_int' => @test_optional_int, - 'some_integer_map' => @some_integer_map, - 'some_record' => @some_record&.as_json, - 'some_optional_record' => @some_optional_record&.as_json, - 'some_record_array' => @some_record_array.map { |v| v&.as_json }, - 'some_record_map' => @some_record_map.transform_values { |v| v&.as_json }, - 'some_enum_array' => @some_enum_array.map { |v| v&.as_json }, - 'some_optional_enum' => @some_optional_enum&.as_json, - 'some_enum_with_default' => @some_enum_with_default&.as_json - } - end - end -end; end - - -spec/app/lib/schema_classes/my_namespace/my_schema_with_date_time.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module MyNamespace - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.MySchemaWithDateTimes - class MySchemaWithDateTime < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [String] - attr_accessor :test_id - # @return [Integer, nil] - attr_accessor :updated_at - # @return [nil, Integer] - attr_accessor :some_int - # @return [nil, Integer] - attr_accessor :some_datetime_int - # @return [String] - attr_accessor :timestamp - - # @override - def initialize(test_id: nil, - updated_at: nil, - some_int: nil, - some_datetime_int: nil, - timestamp: nil) - super - self.test_id = test_id - self.updated_at = updated_at - self.some_int = some_int - self.some_datetime_int = some_datetime_int - self.timestamp = timestamp - end - - # @override - def schema - 'MySchemaWithDateTimes' - end - - # @override - def namespace - 'com.my-namespace' - end - - # @override - def as_json(_opts={}) - { - 'test_id' => @test_id, - 'updated_at' => @updated_at, - 'some_int' => @some_int, - 'some_datetime_int' => @some_datetime_int, - 'timestamp' => @timestamp - } - end - end -end; end - - -spec/app/lib/schema_classes/my_namespace/my_schema_with_id.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module MyNamespace - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.MySchemaWithId - class MySchemaWithId < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [String] - attr_accessor :test_id - # @return [Integer] - attr_accessor :some_int - # @return [String] - attr_accessor :message_id - # @return [String] - attr_accessor :timestamp - - # @override - def initialize(test_id: nil, - some_int: nil, - message_id: nil, - timestamp: nil) - super - self.test_id = test_id - self.some_int = some_int - self.message_id = message_id - self.timestamp = timestamp - end - - # @override - def schema - 'MySchemaWithId' - end - - # @override - def namespace - 'com.my-namespace' - end - - # @override - def as_json(_opts={}) - { - 'test_id' => @test_id, - 'some_int' => @some_int, - 'message_id' => @message_id, - 'timestamp' => @timestamp - } - end - end -end; end - - -spec/app/lib/schema_classes/my_namespace/my_schema_with_unique_id.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module MyNamespace - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.MySchemaWithUniqueId - class MySchemaWithUniqueId < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [Integer] - attr_accessor :id - # @return [String] - attr_accessor :test_id - # @return [Integer] - attr_accessor :some_int - # @return [String] - attr_accessor :message_id - # @return [String] - attr_accessor :timestamp - - # @override - def initialize(id: nil, - test_id: nil, - some_int: nil, - message_id: nil, - timestamp: nil) - super - self.id = id - self.test_id = test_id - self.some_int = some_int - self.message_id = message_id - self.timestamp = timestamp - end - - # @override - def schema - 'MySchemaWithUniqueId' - end - - # @override - def namespace - 'com.my-namespace' - end - - # @override - def as_json(_opts={}) - { - 'id' => @id, - 'test_id' => @test_id, - 'some_int' => @some_int, - 'message_id' => @message_id, - 'timestamp' => @timestamp - } - end - end -end; end - - -spec/app/lib/schema_classes/my_namespace/wibble.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module MyNamespace - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.Wibble - class Wibble < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [Integer] - attr_accessor :id - # @return [Integer] - attr_accessor :wibble_id - # @return [String] - attr_accessor :name - # @return [String] - attr_accessor :floop - # @return [Integer] - attr_accessor :birthday_int - # @return [Integer] - attr_accessor :birthday_long - # @return [nil, Integer] - attr_accessor :birthday_optional - # @return [Integer] - attr_accessor :updated_at - # @return [Integer] - attr_accessor :created_at - - # @override - def initialize(id: nil, - wibble_id: nil, - name: nil, - floop: nil, - birthday_int: nil, - birthday_long: nil, - birthday_optional: nil, - updated_at: nil, - created_at: nil) - super - self.id = id - self.wibble_id = wibble_id - self.name = name - self.floop = floop - self.birthday_int = birthday_int - self.birthday_long = birthday_long - self.birthday_optional = birthday_optional - self.updated_at = updated_at - self.created_at = created_at - end - - # @override - def schema - 'Wibble' - end - - # @override - def namespace - 'com.my-namespace' - end - - # @override - def as_json(_opts={}) - { - 'id' => @id, - 'wibble_id' => @wibble_id, - 'name' => @name, - 'floop' => @floop, - 'birthday_int' => @birthday_int, - 'birthday_long' => @birthday_long, - 'birthday_optional' => @birthday_optional, - 'updated_at' => @updated_at, - 'created_at' => @created_at - } - end - end -end; end - - -spec/app/lib/schema_classes/my_namespace/widget.rb: # frozen_string_literal: true # This file is autogenerated by Deimos, Do NOT modify @@ -1083,297 +61,3 @@ module Schemas; module MyNamespace end end end; end - - -spec/app/lib/schema_classes/my_namespace/widget_the_second.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module MyNamespace - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.WidgetTheSecond - class WidgetTheSecond < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [Integer] - attr_accessor :id - # @return [Integer] - attr_accessor :widget_id - # @return [String] - attr_accessor :model_id - # @return [Integer] - attr_accessor :updated_at - # @return [Integer] - attr_accessor :created_at - - # @override - def initialize(id: nil, - widget_id: nil, - model_id: nil, - updated_at: nil, - created_at: nil) - super - self.id = id - self.widget_id = widget_id - self.model_id = model_id - self.updated_at = updated_at - self.created_at = created_at - end - - # @override - def schema - 'WidgetTheSecond' - end - - # @override - def namespace - 'com.my-namespace' - end - - # @override - def as_json(_opts={}) - { - 'id' => @id, - 'widget_id' => @widget_id, - 'model_id' => @model_id, - 'updated_at' => @updated_at, - 'created_at' => @created_at - } - end - end -end; end - - -spec/app/lib/schema_classes/request/create_topic.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module Request - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.request.CreateTopic - class CreateTopic < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [String] - attr_accessor :request_id - - # @override - def initialize(request_id: nil) - super - self.request_id = request_id - end - - # @override - def schema - 'CreateTopic' - end - - # @override - def namespace - 'com.my-namespace.request' - end - - # @override - def as_json(_opts={}) - { - 'request_id' => @request_id - } - end - end -end; end - - -spec/app/lib/schema_classes/request/index.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module Request - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.request.Index - class Index < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [String] - attr_accessor :request_id - - # @override - def initialize(request_id: nil) - super - self.request_id = request_id - end - - # @override - def schema - 'Index' - end - - # @override - def namespace - 'com.my-namespace.request' - end - - # @override - def as_json(_opts={}) - { - 'request_id' => @request_id - } - end - end -end; end - - -spec/app/lib/schema_classes/request/update_request.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module Request - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.request.UpdateRequest - class UpdateRequest < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [String] - attr_accessor :update_request_id - - # @override - def initialize(update_request_id: nil) - super - self.update_request_id = update_request_id - end - - # @override - def schema - 'UpdateRequest' - end - - # @override - def namespace - 'com.my-namespace.request' - end - - # @override - def as_json(_opts={}) - { - 'update_request_id' => @update_request_id - } - end - end -end; end - - -spec/app/lib/schema_classes/response/create_topic.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module Response - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.response.CreateTopic - class CreateTopic < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [String] - attr_accessor :response_id - - # @override - def initialize(response_id: nil) - super - self.response_id = response_id - end - - # @override - def schema - 'CreateTopic' - end - - # @override - def namespace - 'com.my-namespace.response' - end - - # @override - def as_json(_opts={}) - { - 'response_id' => @response_id - } - end - end -end; end - - -spec/app/lib/schema_classes/response/index.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module Response - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.response.Index - class Index < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [String] - attr_accessor :response_id - - # @override - def initialize(response_id: nil) - super - self.response_id = response_id - end - - # @override - def schema - 'Index' - end - - # @override - def namespace - 'com.my-namespace.response' - end - - # @override - def as_json(_opts={}) - { - 'response_id' => @response_id - } - end - end -end; end - - -spec/app/lib/schema_classes/response/update_response.rb: -# frozen_string_literal: true - -# This file is autogenerated by Deimos, Do NOT modify -module Schemas; module Response - ### Primary Schema Class ### - # Autogenerated Schema for Record at com.my-namespace.response.UpdateResponse - class UpdateResponse < Deimos::SchemaClass::Record - - ### Attribute Accessors ### - # @return [String] - attr_accessor :update_response_id - - # @override - def initialize(update_response_id: nil) - super - self.update_response_id = update_response_id - end - - # @override - def schema - 'UpdateResponse' - end - - # @override - def namespace - 'com.my-namespace.response' - end - - # @override - def as_json(_opts={}) - { - 'update_response_id' => @update_response_id - } - end - end -end; end - diff --git a/spec/snapshots/widget-skip-generate-from-schema-files.snap b/spec/snapshots/widget-skip-generate-from-schema-files.snap new file mode 100644 index 00000000..736c0df4 --- /dev/null +++ b/spec/snapshots/widget-skip-generate-from-schema-files.snap @@ -0,0 +1,65 @@ +spec/app/lib/schema_classes/widget.rb: +# frozen_string_literal: true + +# This file is autogenerated by Deimos, Do NOT modify +module Schemas + ### Primary Schema Class ### + # Autogenerated Schema for Record at com.my-namespace.Widget + class Widget < Deimos::SchemaClass::Record + + ### Attribute Accessors ### + # @return [Integer] + attr_accessor :id + # @return [Integer] + attr_accessor :widget_id + # @return [String] + attr_accessor :name + # @return [Integer] + attr_accessor :updated_at + # @return [Integer] + attr_accessor :created_at + + # @override + def initialize(id: nil, + widget_id: nil, + name: nil, + updated_at: nil, + created_at: nil) + super + self.id = id + self.widget_id = widget_id + self.name = name + self.updated_at = updated_at + self.created_at = created_at + end + + # @override + def schema + 'Widget' + end + + # @override + def namespace + 'com.my-namespace' + end + + def self.tombstone(key) + record = self.allocate + record.tombstone_key = key + record.a_string = key + record + end + + # @override + def as_json(_opts={}) + { + 'id' => @id, + 'widget_id' => @widget_id, + 'name' => @name, + 'updated_at' => @updated_at, + 'created_at' => @created_at + } + end + end +end + From 3e414782b6054c0f110d8a040942c720451c3e4c Mon Sep 17 00:00:00 2001 From: jenchanflipp <72824964+jenchanflipp@users.noreply.github.com> Date: Wed, 25 Jan 2023 16:00:51 -0500 Subject: [PATCH 4/7] GUILD-690: Validate if passed in config file exists and only generate deimos consumer config --- .../templates/config.rb.tt | 29 +-------- .../templates/consumer_config.rb.tt | 12 ++++ .../active_record_consumer_generator.rb | 58 ++++++++++-------- .../active_record_consumer_generator_spec.rb | 59 ++++++++++++++++--- spec/snapshots/consumer_generator_config.snap | 15 +---- ...umer_generator_config_with_config_arg.snap | 16 +---- ...umer_generator_existing_deimos_config.snap | 15 +++++ .../consumer_generator_new_deimos_config.snap | 16 +++++ 8 files changed, 133 insertions(+), 87 deletions(-) create mode 100644 lib/generators/deimos/active_record_consumer/templates/consumer_config.rb.tt create mode 100644 spec/snapshots/consumer_generator_existing_deimos_config.snap create mode 100644 spec/snapshots/consumer_generator_new_deimos_config.snap diff --git a/lib/generators/deimos/active_record_consumer/templates/config.rb.tt b/lib/generators/deimos/active_record_consumer/templates/config.rb.tt index f507161f..a89d0ad8 100644 --- a/lib/generators/deimos/active_record_consumer/templates/config.rb.tt +++ b/lib/generators/deimos/active_record_consumer/templates/config.rb.tt @@ -1,29 +1,4 @@ - Deimos.configure do - logger Logger.new(STDOUT) - # Nested config field - kafka.seed_brokers ['my.kafka.broker:9092'] - - schema.generate_namespace_folders true - schema.generated_class_path 'app/lib/schemas' - - # Multiple nested config fields via block - consumers do - session_timeout 30 - offset_commit_interval 10 - end - - # Define a consumer - consumer do - class_name '<%= consumer_name.classify %>' - topic 'TopicToConsume' - schema '<%= schema %>' - namespace '<%= namespace %>' - key_config field: :<%= key_field %> - # include Phobos / RubyKafka configs - start_from_beginning true - heartbeat_interval 10 - use_schema_classes true - end - +<%= @consumer_config %> end + diff --git a/lib/generators/deimos/active_record_consumer/templates/consumer_config.rb.tt b/lib/generators/deimos/active_record_consumer/templates/consumer_config.rb.tt new file mode 100644 index 00000000..a93da135 --- /dev/null +++ b/lib/generators/deimos/active_record_consumer/templates/consumer_config.rb.tt @@ -0,0 +1,12 @@ + # Define a consumer + consumer do + class_name '<%= consumer_name.classify %>' + topic 'TopicToConsume' + schema '<%= schema %>' + namespace '<%= namespace %>' + key_config field: :<%= key_field %> + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end diff --git a/lib/generators/deimos/active_record_consumer_generator.rb b/lib/generators/deimos/active_record_consumer_generator.rb index 238906d3..89cba01f 100644 --- a/lib/generators/deimos/active_record_consumer_generator.rb +++ b/lib/generators/deimos/active_record_consumer_generator.rb @@ -40,23 +40,11 @@ class ActiveRecordConsumerGenerator < Rails::Generators::Base no_commands do + # validate schema, key_config and deimos config file def validate_arguments - # key_config_hash = {} - # colon = self.full_schema.rindex(':') - # value = key_config[colon+1...-1].strip - # if value.rindex("'").present? - # key_config_hash[key_config[0...colon]] = key_config[colon+1...-1] - # - # self.full_schema[0...last_dot] - - @key_config_hash = {} - if KEY_CONFIG_OPTIONS_BOOL.include?(key_config_type) - @key_config_hash[key_config_type] = true - elsif KEY_CONFIG_OPTIONS_STRING.include?(key_config_type) - @key_config_hash[key_config_type] = key_config_value - else - return ' nil' - end + _validate_schema + _validate_key_config + _validate_config_path end # Creates database migration for creating new table and Rails Model @@ -72,15 +60,19 @@ def create_consumer # Adds consumer config to config file. # Defaults to deimos.rb if config_path arg is not specified. def create_consumer_config - config_file = 'deimos.rb' - config_file_path = "#{initializer_path}/#{config_file}" - config_file_path = config_path if config_path.present? + if @config_file_path.nil? + config_file = 'deimos.rb' + @config_file_path = "#{initializer_path}/#{config_file}" + end - if File.exist?(config_file_path) - config_template = File.expand_path(find_in_source_paths('config.rb')) - insert_into_file(config_file_path.to_s, CapturableERB.new(::File.binread(config_template)).result(binding)) + config_template = File.expand_path(find_in_source_paths('consumer_config.rb')) + if File.exist?(@config_file_path) + insert_into_file(@config_file_path.to_s, + CapturableERB.new(::File.binread(config_template)).result(binding), + :after => "Deimos.configure do\n") else - template('config.rb', config_file_path.to_s) + @consumer_config = CapturableERB.new(::File.binread(config_template)).result(binding) + template('config.rb', @config_file_path.to_s) end end @@ -160,10 +152,28 @@ def generate # Determines if Schema Class Generation can be run. # @raise if Schema Backend is not of a Avro-based class - def _validate + def _validate_schema backend = Deimos.config.schema.backend.to_s raise 'Schema Class Generation requires an Avro-based Schema Backend' if backend !~ /^avro/ end + + def _validate_key_config + @key_config_hash = {} + if KEY_CONFIG_OPTIONS_BOOL.include?(key_config_type) + @key_config_hash[key_config_type] = true + elsif KEY_CONFIG_OPTIONS_STRING.include?(key_config_type) + @key_config_hash[key_config_type] = key_config_value + else + return ' nil' + end + end + + def _validate_config_path + if config_path.present? + @config_file_path = "#{initializer_path}/#{config_path}" + raise 'Configuration file does not exist!' unless File.exist?(@config_file_path) + end + end end end end diff --git a/spec/generators/active_record_consumer_generator_spec.rb b/spec/generators/active_record_consumer_generator_spec.rb index ea67339b..c09182a2 100644 --- a/spec/generators/active_record_consumer_generator_spec.rb +++ b/spec/generators/active_record_consumer_generator_spec.rb @@ -15,6 +15,7 @@ FileUtils.rm_rf('db') if File.exist?('db') FileUtils.rm_rf('app') if File.exist?('app') FileUtils.rm_rf('config') if File.exist?('config') + FileUtils.rm_rf('spec/app') if File.exist?('spec/app') end before(:each) do @@ -27,7 +28,51 @@ end end - it 'should generate a migration, model, consumer class, config and schema class' do + it 'should generate a migration, model, consumer class, config and schema class with existing deimos.rb config file' do + config_file_name = 'deimos.rb' + FileUtils.mkdir_p(config_path) + deimos_file_path = "#{config_path}/#{config_file_name}" + File.new(deimos_file_path, "w") + File.open(deimos_file_path, "w") { |f| f.write "Deimos.configure do\n\nend" } + + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end + end + + expect(Dir["#{db_migration_path}/*.rb"]).to be_empty + expect(Dir["#{model_path}/*.rb"]).to be_empty + expect(Dir["#{schema_class_path}/*.rb"]).to be_empty + + described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key']) + + files = Dir["#{db_migration_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_migration') + + files = Dir["#{model_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_model') + + files = Dir["#{consumer_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_consumer_class') + + files = Dir["#{config_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_existing_deimos_config') + + files = Dir["#{schema_class_path}/*/*.rb"] + expect(File.read(files[0])).to match_snapshot('consumer_generator_schema_classes') + end + + it 'should generate a migration, model, consumer class, config and schema class with no existing deimos.rb configuration file' do + Deimos.configure do consumer do class_name 'ConsumerTest::MyConsumer' @@ -40,7 +85,6 @@ expect(Dir["#{db_migration_path}/*.rb"]).to be_empty expect(Dir["#{model_path}/*.rb"]).to be_empty - expect(Dir["#{config_path}/*.rb"]).to be_empty expect(Dir["#{schema_class_path}/*.rb"]).to be_empty described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key']) @@ -59,7 +103,7 @@ files = Dir["#{config_path}/*.rb"] expect(files.length).to eq(1) - expect(File.read(files[0])).to match_snapshot('consumer_generator_config') + expect(File.read(files[0])).to match_snapshot('consumer_generator_new_deimos_config') files = Dir["#{schema_class_path}/*/*.rb"] expect(File.read(files[0])).to match_snapshot('consumer_generator_schema_classes') @@ -68,8 +112,9 @@ it 'should generate a migration, model, consumer class, schema class and edit the passed in config file' do config_file_name = 'my_config.config' FileUtils.mkdir_p(config_path) - Tempfile.new(config_file_name, config_path) - config_path_argument = "#{config_path}/#{config_file_name}" + my_config_file_path = "#{config_path}/#{config_file_name}" + File.new(my_config_file_path, "w") + File.open(my_config_file_path, "w") { |f| f.write "Deimos.configure do\n\nend" } Deimos.configure do consumer do @@ -85,7 +130,7 @@ expect(Dir["#{model_path}/*.rb"]).to be_empty expect(Dir["#{schema_class_path}/*.rb"]).to be_empty - described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key',config_path_argument]) + described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key',config_file_name]) files = Dir["#{db_migration_path}/*.rb"] expect(files.length).to eq(1) @@ -99,7 +144,7 @@ expect(files.length).to eq(1) expect(File.read(files[0])).to match_snapshot('consumer_generator_consumer_class') - expect(File.read(config_path_argument)).to match_snapshot('consumer_generator_config_with_config_arg') + expect(File.read(my_config_file_path)).to match_snapshot('consumer_generator_config_with_config_arg') files = Dir["#{schema_class_path}/*/*.rb"] expect(File.read(files[0])).to match_snapshot('consumer_generator_schema_classes') diff --git a/spec/snapshots/consumer_generator_config.snap b/spec/snapshots/consumer_generator_config.snap index 40ed5d45..37655f09 100644 --- a/spec/snapshots/consumer_generator_config.snap +++ b/spec/snapshots/consumer_generator_config.snap @@ -1,18 +1,4 @@ - Deimos.configure do - logger Logger.new(STDOUT) - # Nested config field - kafka.seed_brokers ['my.kafka.broker:9092'] - - schema.generate_namespace_folders true - schema.generated_class_path 'app/lib/schemas' - - # Multiple nested config fields via block - consumers do - session_timeout 30 - offset_commit_interval 10 - end - # Define a consumer consumer do class_name 'WidgetConsumer' @@ -27,3 +13,4 @@ Deimos.configure do end end + diff --git a/spec/snapshots/consumer_generator_config_with_config_arg.snap b/spec/snapshots/consumer_generator_config_with_config_arg.snap index 40ed5d45..c2cc2014 100644 --- a/spec/snapshots/consumer_generator_config_with_config_arg.snap +++ b/spec/snapshots/consumer_generator_config_with_config_arg.snap @@ -1,18 +1,4 @@ - Deimos.configure do - logger Logger.new(STDOUT) - # Nested config field - kafka.seed_brokers ['my.kafka.broker:9092'] - - schema.generate_namespace_folders true - schema.generated_class_path 'app/lib/schemas' - - # Multiple nested config fields via block - consumers do - session_timeout 30 - offset_commit_interval 10 - end - # Define a consumer consumer do class_name 'WidgetConsumer' @@ -26,4 +12,4 @@ Deimos.configure do use_schema_classes true end -end +end \ No newline at end of file diff --git a/spec/snapshots/consumer_generator_existing_deimos_config.snap b/spec/snapshots/consumer_generator_existing_deimos_config.snap new file mode 100644 index 00000000..c2cc2014 --- /dev/null +++ b/spec/snapshots/consumer_generator_existing_deimos_config.snap @@ -0,0 +1,15 @@ +Deimos.configure do + # Define a consumer + consumer do + class_name 'WidgetConsumer' + topic 'TopicToConsume' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :id + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end + +end \ No newline at end of file diff --git a/spec/snapshots/consumer_generator_new_deimos_config.snap b/spec/snapshots/consumer_generator_new_deimos_config.snap new file mode 100644 index 00000000..37655f09 --- /dev/null +++ b/spec/snapshots/consumer_generator_new_deimos_config.snap @@ -0,0 +1,16 @@ +Deimos.configure do + # Define a consumer + consumer do + class_name 'WidgetConsumer' + topic 'TopicToConsume' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :id + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end + +end + From 1222c740a32ea0fced434593601555bb7ae7da0c Mon Sep 17 00:00:00 2001 From: jenchanflipp <72824964+jenchanflipp@users.noreply.github.com> Date: Wed, 25 Jan 2023 16:58:32 -0500 Subject: [PATCH 5/7] GUILD-690: Handling config file edge cases --- .../active_record_consumer_generator.rb | 18 +- .../active_record_consumer_generator_spec.rb | 197 +++++++++--------- .../consumer_generator_config_arg.snap | 15 ++ ...r_config_arg_missing_deimos_configure.snap | 16 ++ 4 files changed, 145 insertions(+), 101 deletions(-) create mode 100644 spec/snapshots/consumer_generator_config_arg.snap create mode 100644 spec/snapshots/consumer_generator_config_arg_missing_deimos_configure.snap diff --git a/lib/generators/deimos/active_record_consumer_generator.rb b/lib/generators/deimos/active_record_consumer_generator.rb index 89cba01f..33873398 100644 --- a/lib/generators/deimos/active_record_consumer_generator.rb +++ b/lib/generators/deimos/active_record_consumer_generator.rb @@ -65,13 +65,21 @@ def create_consumer_config @config_file_path = "#{initializer_path}/#{config_file}" end - config_template = File.expand_path(find_in_source_paths('consumer_config.rb')) + consumer_config_template = File.expand_path(find_in_source_paths('consumer_config.rb')) if File.exist?(@config_file_path) - insert_into_file(@config_file_path.to_s, - CapturableERB.new(::File.binread(config_template)).result(binding), - :after => "Deimos.configure do\n") + # if file has Deimos.configure statement then add consumers block after it + if File.readlines(@config_file_path).grep(/Deimos.configure do/).size > 0 + insert_into_file(@config_file_path.to_s, + CapturableERB.new(::File.binread(consumer_config_template)).result(binding), + :after => "Deimos.configure do\n") + else + # if file does not have Deimos.configure statement then add it plus consumers block + @consumer_config = CapturableERB.new(::File.binread(consumer_config_template)).result(binding) + config_template = File.expand_path(find_in_source_paths('config.rb')) + insert_into_file(@config_file_path.to_s,CapturableERB.new(::File.binread(config_template)).result(binding)) + end else - @consumer_config = CapturableERB.new(::File.binread(config_template)).result(binding) + @consumer_config = CapturableERB.new(::File.binread(consumer_config_template)).result(binding) template('config.rb', @config_file_path.to_s) end end diff --git a/spec/generators/active_record_consumer_generator_spec.rb b/spec/generators/active_record_consumer_generator_spec.rb index c09182a2..e7f49da4 100644 --- a/spec/generators/active_record_consumer_generator_spec.rb +++ b/spec/generators/active_record_consumer_generator_spec.rb @@ -28,126 +28,131 @@ end end - it 'should generate a migration, model, consumer class, config and schema class with existing deimos.rb config file' do - config_file_name = 'deimos.rb' - FileUtils.mkdir_p(config_path) - deimos_file_path = "#{config_path}/#{config_file_name}" - File.new(deimos_file_path, "w") - File.open(deimos_file_path, "w") { |f| f.write "Deimos.configure do\n\nend" } - - Deimos.configure do - consumer do - class_name 'ConsumerTest::MyConsumer' - topic 'MyTopic' - schema 'Widget' - namespace 'com.my-namespace' - key_config field: :a_string + context 'with a regular flow' do + + it 'should generate a migration, model, consumer class, config and schema class with existing deimos.rb config file' do + config_file_name = 'deimos.rb' + FileUtils.mkdir_p(config_path) + deimos_file_path = "#{config_path}/#{config_file_name}" + File.new(deimos_file_path, "w") + File.open(deimos_file_path, "w") { |f| f.write "Deimos.configure do\n\nend" } + + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end end - end - - expect(Dir["#{db_migration_path}/*.rb"]).to be_empty - expect(Dir["#{model_path}/*.rb"]).to be_empty - expect(Dir["#{schema_class_path}/*.rb"]).to be_empty - - described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key']) - files = Dir["#{db_migration_path}/*.rb"] - expect(files.length).to eq(1) - expect(File.read(files[0])).to match_snapshot('consumer_generator_migration') + expect(Dir["#{db_migration_path}/*.rb"]).to be_empty + expect(Dir["#{model_path}/*.rb"]).to be_empty + expect(Dir["#{schema_class_path}/*.rb"]).to be_empty - files = Dir["#{model_path}/*.rb"] - expect(files.length).to eq(1) - expect(File.read(files[0])).to match_snapshot('consumer_generator_model') + described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key']) - files = Dir["#{consumer_path}/*.rb"] - expect(files.length).to eq(1) - expect(File.read(files[0])).to match_snapshot('consumer_generator_consumer_class') + files = Dir["#{db_migration_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_migration') - files = Dir["#{config_path}/*.rb"] - expect(files.length).to eq(1) - expect(File.read(files[0])).to match_snapshot('consumer_generator_existing_deimos_config') + files = Dir["#{model_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_model') - files = Dir["#{schema_class_path}/*/*.rb"] - expect(File.read(files[0])).to match_snapshot('consumer_generator_schema_classes') - end + files = Dir["#{consumer_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_consumer_class') - it 'should generate a migration, model, consumer class, config and schema class with no existing deimos.rb configuration file' do + files = Dir["#{config_path}/*.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_existing_deimos_config') - Deimos.configure do - consumer do - class_name 'ConsumerTest::MyConsumer' - topic 'MyTopic' - schema 'Widget' - namespace 'com.my-namespace' - key_config field: :a_string - end + files = Dir["#{schema_class_path}/*/*.rb"] + expect(File.read(files[0])).to match_snapshot('consumer_generator_schema_classes') end - expect(Dir["#{db_migration_path}/*.rb"]).to be_empty - expect(Dir["#{model_path}/*.rb"]).to be_empty - expect(Dir["#{schema_class_path}/*.rb"]).to be_empty - - described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key']) + end - files = Dir["#{db_migration_path}/*.rb"] - expect(files.length).to eq(1) - expect(File.read(files[0])).to match_snapshot('consumer_generator_migration') - files = Dir["#{model_path}/*.rb"] - expect(files.length).to eq(1) - expect(File.read(files[0])).to match_snapshot('consumer_generator_model') + context 'with different config file input' do + it 'should generate correct config with no existing deimos.rb configuration file' do - files = Dir["#{consumer_path}/*.rb"] - expect(files.length).to eq(1) - expect(File.read(files[0])).to match_snapshot('consumer_generator_consumer_class') + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end + end - files = Dir["#{config_path}/*.rb"] - expect(files.length).to eq(1) - expect(File.read(files[0])).to match_snapshot('consumer_generator_new_deimos_config') + expect(Dir["#{db_migration_path}/*.rb"]).to be_empty + expect(Dir["#{model_path}/*.rb"]).to be_empty + expect(Dir["#{schema_class_path}/*.rb"]).to be_empty - files = Dir["#{schema_class_path}/*/*.rb"] - expect(File.read(files[0])).to match_snapshot('consumer_generator_schema_classes') - end + described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key']) - it 'should generate a migration, model, consumer class, schema class and edit the passed in config file' do - config_file_name = 'my_config.config' - FileUtils.mkdir_p(config_path) - my_config_file_path = "#{config_path}/#{config_file_name}" - File.new(my_config_file_path, "w") - File.open(my_config_file_path, "w") { |f| f.write "Deimos.configure do\n\nend" } + files = Dir["#{config_path}/deimos.rb"] + expect(files.length).to eq(1) + expect(File.read(files[0])).to match_snapshot('consumer_generator_new_deimos_config') + end - Deimos.configure do - consumer do - class_name 'ConsumerTest::MyConsumer' - topic 'MyTopic' - schema 'Widget' - namespace 'com.my-namespace' - key_config field: :a_string + it 'should generate correct config for the passed in config file that contains Deimos.configure' do + config_file_name = 'my_config.config' + FileUtils.mkdir_p(config_path) + my_config_file_path = "#{config_path}/#{config_file_name}" + File.new(my_config_file_path, "w") + File.open(my_config_file_path, "w") { |f| f.write "Deimos.configure do\n\nend" } + + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end end - end - expect(Dir["#{db_migration_path}/*.rb"]).to be_empty - expect(Dir["#{model_path}/*.rb"]).to be_empty - expect(Dir["#{schema_class_path}/*.rb"]).to be_empty + expect(Dir["#{db_migration_path}/*.rb"]).to be_empty + expect(Dir["#{model_path}/*.rb"]).to be_empty + expect(Dir["#{schema_class_path}/*.rb"]).to be_empty - described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key',config_file_name]) + described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key',config_file_name]) - files = Dir["#{db_migration_path}/*.rb"] - expect(files.length).to eq(1) - expect(File.read(files[0])).to match_snapshot('consumer_generator_migration') + files = Dir["#{config_path}/*.config"] + expect(files.length).to eq(1) + expect(File.read(my_config_file_path)).to match_snapshot('consumer_generator_config_arg') + end - files = Dir["#{model_path}/*.rb"] - expect(files.length).to eq(1) - expect(File.read(files[0])).to match_snapshot('consumer_generator_model') + it 'should generate correct config for the passed in config file with missing Deimos.configure' do + config_file_name = 'my_config.config' + FileUtils.mkdir_p(config_path) + my_config_file_path = "#{config_path}/#{config_file_name}" + File.new(my_config_file_path, "w") + + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end + end - files = Dir["#{consumer_path}/*.rb"] - expect(files.length).to eq(1) - expect(File.read(files[0])).to match_snapshot('consumer_generator_consumer_class') + expect(Dir["#{db_migration_path}/*.rb"]).to be_empty + expect(Dir["#{model_path}/*.rb"]).to be_empty + expect(Dir["#{schema_class_path}/*.rb"]).to be_empty - expect(File.read(my_config_file_path)).to match_snapshot('consumer_generator_config_with_config_arg') + described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key',config_file_name]) - files = Dir["#{schema_class_path}/*/*.rb"] - expect(File.read(files[0])).to match_snapshot('consumer_generator_schema_classes') + files = Dir["#{config_path}/*.config"] + expect(files.length).to eq(1) + expect(File.read(my_config_file_path)).to match_snapshot('consumer_generator_config_arg_missing_deimos_configure') + end end - end diff --git a/spec/snapshots/consumer_generator_config_arg.snap b/spec/snapshots/consumer_generator_config_arg.snap new file mode 100644 index 00000000..c2cc2014 --- /dev/null +++ b/spec/snapshots/consumer_generator_config_arg.snap @@ -0,0 +1,15 @@ +Deimos.configure do + # Define a consumer + consumer do + class_name 'WidgetConsumer' + topic 'TopicToConsume' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :id + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end + +end \ No newline at end of file diff --git a/spec/snapshots/consumer_generator_config_arg_missing_deimos_configure.snap b/spec/snapshots/consumer_generator_config_arg_missing_deimos_configure.snap new file mode 100644 index 00000000..37655f09 --- /dev/null +++ b/spec/snapshots/consumer_generator_config_arg_missing_deimos_configure.snap @@ -0,0 +1,16 @@ +Deimos.configure do + # Define a consumer + consumer do + class_name 'WidgetConsumer' + topic 'TopicToConsume' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :id + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end + +end + From 1ee26155530c72bd760d88fb4a1382c58f7a27d4 Mon Sep 17 00:00:00 2001 From: jenchanflipp <72824964+jenchanflipp@users.noreply.github.com> Date: Wed, 25 Jan 2023 17:38:39 -0500 Subject: [PATCH 6/7] GUILD-690: Add support for specifying key_config arg --- .../templates/consumer_config.rb.tt | 2 +- .../active_record_consumer_generator.rb | 20 ++-- .../active_record_consumer_generator_spec.rb | 91 ++++++++++++++++++- .../consumer_generator_config_arg.snap | 2 +- ...r_config_arg_missing_deimos_configure.snap | 2 +- ...umer_generator_existing_deimos_config.snap | 2 +- .../consumer_generator_key_config_none.snap | 16 ++++ .../consumer_generator_key_config_plain.snap | 16 ++++ .../consumer_generator_key_config_schema.snap | 16 ++++ .../consumer_generator_new_deimos_config.snap | 2 +- 10 files changed, 152 insertions(+), 17 deletions(-) create mode 100644 spec/snapshots/consumer_generator_key_config_none.snap create mode 100644 spec/snapshots/consumer_generator_key_config_plain.snap create mode 100644 spec/snapshots/consumer_generator_key_config_schema.snap diff --git a/lib/generators/deimos/active_record_consumer/templates/consumer_config.rb.tt b/lib/generators/deimos/active_record_consumer/templates/consumer_config.rb.tt index a93da135..ae8492c8 100644 --- a/lib/generators/deimos/active_record_consumer/templates/consumer_config.rb.tt +++ b/lib/generators/deimos/active_record_consumer/templates/consumer_config.rb.tt @@ -4,7 +4,7 @@ topic 'TopicToConsume' schema '<%= schema %>' namespace '<%= namespace %>' - key_config field: :<%= key_field %> + key_config <%= @key_type %>: :<%= @key_value %> # include Phobos / RubyKafka configs start_from_beginning true heartbeat_interval 10 diff --git a/lib/generators/deimos/active_record_consumer_generator.rb b/lib/generators/deimos/active_record_consumer_generator.rb index 33873398..8c2c2b0f 100644 --- a/lib/generators/deimos/active_record_consumer_generator.rb +++ b/lib/generators/deimos/active_record_consumer_generator.rb @@ -136,10 +136,10 @@ def initializer_path end end - # @return [String] Returns the name of the first field in the schema, as the key - def key_field - fields.first.name - end + # # @return [String] Returns the name of the first field in the schema, as the key + # def key_field + # fields.first.name + # end end # desc 'Generate necessary files and configuration for a new Active Record Consumer.' @@ -165,14 +165,18 @@ def _validate_schema raise 'Schema Class Generation requires an Avro-based Schema Backend' if backend !~ /^avro/ end +# key_config none: true - this indicates that you are not using keys at all for this topic. This must be set if your messages won't have keys - either all your messages in a topic need to have a key, or they all need to have no key. This is a good choice for events that aren't keyed - you can still set a partition key. +# key_config plain: true - this indicates that you are not using an encoded key. Use this for legacy topics - new topics should not use this setting. +# key_config schema: 'MyKeySchema-key' - this tells the producer to look for an existing key schema named MyKeySchema-key in the schema registry and to encode the key using it. Use this if you've already created a key schema or the key value does not exist in the existing payload (e.g. it is a compound or generated key). +# key_config field: 'my_field' - this tells the producer to look for a field named my_field in the value schema. When a payload comes in, the producer will take that value from the payload and insert it in a dynamically generated key schema. This key schema does not need to live in your codebase. Instead, it will be a subset of the value schema with only the key field in it. def _validate_key_config - @key_config_hash = {} + @key_type = key_config_type if KEY_CONFIG_OPTIONS_BOOL.include?(key_config_type) - @key_config_hash[key_config_type] = true + @key_value = 'true' elsif KEY_CONFIG_OPTIONS_STRING.include?(key_config_type) - @key_config_hash[key_config_type] = key_config_value + @key_value = key_config_value else - return ' nil' + raise 'Invalid key config specified!' end end diff --git a/spec/generators/active_record_consumer_generator_spec.rb b/spec/generators/active_record_consumer_generator_spec.rb index e7f49da4..25d3a167 100644 --- a/spec/generators/active_record_consumer_generator_spec.rb +++ b/spec/generators/active_record_consumer_generator_spec.rb @@ -51,7 +51,7 @@ expect(Dir["#{model_path}/*.rb"]).to be_empty expect(Dir["#{schema_class_path}/*.rb"]).to be_empty - described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key']) + described_class.start(['com.my-namespace.Widget','field','a_string']) files = Dir["#{db_migration_path}/*.rb"] expect(files.length).to eq(1) @@ -93,7 +93,7 @@ expect(Dir["#{model_path}/*.rb"]).to be_empty expect(Dir["#{schema_class_path}/*.rb"]).to be_empty - described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key']) + described_class.start(['com.my-namespace.Widget','field','a-string']) files = Dir["#{config_path}/deimos.rb"] expect(files.length).to eq(1) @@ -121,7 +121,7 @@ expect(Dir["#{model_path}/*.rb"]).to be_empty expect(Dir["#{schema_class_path}/*.rb"]).to be_empty - described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key',config_file_name]) + described_class.start(['com.my-namespace.Widget','field','a-string',config_file_name]) files = Dir["#{config_path}/*.config"] expect(files.length).to eq(1) @@ -148,11 +148,94 @@ expect(Dir["#{model_path}/*.rb"]).to be_empty expect(Dir["#{schema_class_path}/*.rb"]).to be_empty - described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key',config_file_name]) + described_class.start(['com.my-namespace.Widget','field','a-string',config_file_name]) files = Dir["#{config_path}/*.config"] expect(files.length).to eq(1) expect(File.read(my_config_file_path)).to match_snapshot('consumer_generator_config_arg_missing_deimos_configure') end end + + context 'with different key schema arguments specified' do + it 'should generate correct key_config when specifying key_config none: true' do + config_file_name = 'my_config.config' + FileUtils.mkdir_p(config_path) + my_config_file_path = "#{config_path}/#{config_file_name}" + File.new(my_config_file_path, "w") + + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end + end + + expect(Dir["#{db_migration_path}/*.rb"]).to be_empty + expect(Dir["#{model_path}/*.rb"]).to be_empty + expect(Dir["#{schema_class_path}/*.rb"]).to be_empty + + described_class.start(['com.my-namespace.Widget','none','true',config_file_name]) + + files = Dir["#{config_path}/*.config"] + expect(files.length).to eq(1) + expect(File.read(my_config_file_path)).to match_snapshot('consumer_generator_key_config_none') + end + + it 'should generate correct key_config when specifying key_config plain: true' do + config_file_name = 'my_config.config' + FileUtils.mkdir_p(config_path) + my_config_file_path = "#{config_path}/#{config_file_name}" + File.new(my_config_file_path, "w") + + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end + end + + expect(Dir["#{db_migration_path}/*.rb"]).to be_empty + expect(Dir["#{model_path}/*.rb"]).to be_empty + expect(Dir["#{schema_class_path}/*.rb"]).to be_empty + + described_class.start(['com.my-namespace.Widget','plain','true',config_file_name]) + + files = Dir["#{config_path}/*.config"] + expect(files.length).to eq(1) + expect(File.read(my_config_file_path)).to match_snapshot('consumer_generator_key_config_plain') + end + + it 'should generate correct key_config when specifying key_config schema: MyKeySchema-key' do + config_file_name = 'my_config.config' + FileUtils.mkdir_p(config_path) + my_config_file_path = "#{config_path}/#{config_file_name}" + File.new(my_config_file_path, "w") + + Deimos.configure do + consumer do + class_name 'ConsumerTest::MyConsumer' + topic 'MyTopic' + schema 'Widget' + namespace 'com.my-namespace' + key_config field: :a_string + end + end + + expect(Dir["#{db_migration_path}/*.rb"]).to be_empty + expect(Dir["#{model_path}/*.rb"]).to be_empty + expect(Dir["#{schema_class_path}/*.rb"]).to be_empty + + described_class.start(['com.my-namespace.Widget','schema','MyKeySchema-key',config_file_name]) + + files = Dir["#{config_path}/*.config"] + expect(files.length).to eq(1) + expect(File.read(my_config_file_path)).to match_snapshot('consumer_generator_key_config_schema') + end + end end diff --git a/spec/snapshots/consumer_generator_config_arg.snap b/spec/snapshots/consumer_generator_config_arg.snap index c2cc2014..c8fc0481 100644 --- a/spec/snapshots/consumer_generator_config_arg.snap +++ b/spec/snapshots/consumer_generator_config_arg.snap @@ -5,7 +5,7 @@ Deimos.configure do topic 'TopicToConsume' schema 'Widget' namespace 'com.my-namespace' - key_config field: :id + key_config field: :a-string # include Phobos / RubyKafka configs start_from_beginning true heartbeat_interval 10 diff --git a/spec/snapshots/consumer_generator_config_arg_missing_deimos_configure.snap b/spec/snapshots/consumer_generator_config_arg_missing_deimos_configure.snap index 37655f09..c132e405 100644 --- a/spec/snapshots/consumer_generator_config_arg_missing_deimos_configure.snap +++ b/spec/snapshots/consumer_generator_config_arg_missing_deimos_configure.snap @@ -5,7 +5,7 @@ Deimos.configure do topic 'TopicToConsume' schema 'Widget' namespace 'com.my-namespace' - key_config field: :id + key_config field: :a-string # include Phobos / RubyKafka configs start_from_beginning true heartbeat_interval 10 diff --git a/spec/snapshots/consumer_generator_existing_deimos_config.snap b/spec/snapshots/consumer_generator_existing_deimos_config.snap index c2cc2014..c8cc12b2 100644 --- a/spec/snapshots/consumer_generator_existing_deimos_config.snap +++ b/spec/snapshots/consumer_generator_existing_deimos_config.snap @@ -5,7 +5,7 @@ Deimos.configure do topic 'TopicToConsume' schema 'Widget' namespace 'com.my-namespace' - key_config field: :id + key_config field: :a_string # include Phobos / RubyKafka configs start_from_beginning true heartbeat_interval 10 diff --git a/spec/snapshots/consumer_generator_key_config_none.snap b/spec/snapshots/consumer_generator_key_config_none.snap new file mode 100644 index 00000000..fc37d5a2 --- /dev/null +++ b/spec/snapshots/consumer_generator_key_config_none.snap @@ -0,0 +1,16 @@ +Deimos.configure do + # Define a consumer + consumer do + class_name 'WidgetConsumer' + topic 'TopicToConsume' + schema 'Widget' + namespace 'com.my-namespace' + key_config none: :true + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end + +end + diff --git a/spec/snapshots/consumer_generator_key_config_plain.snap b/spec/snapshots/consumer_generator_key_config_plain.snap new file mode 100644 index 00000000..d210c7c6 --- /dev/null +++ b/spec/snapshots/consumer_generator_key_config_plain.snap @@ -0,0 +1,16 @@ +Deimos.configure do + # Define a consumer + consumer do + class_name 'WidgetConsumer' + topic 'TopicToConsume' + schema 'Widget' + namespace 'com.my-namespace' + key_config plain: :true + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end + +end + diff --git a/spec/snapshots/consumer_generator_key_config_schema.snap b/spec/snapshots/consumer_generator_key_config_schema.snap new file mode 100644 index 00000000..4c703e67 --- /dev/null +++ b/spec/snapshots/consumer_generator_key_config_schema.snap @@ -0,0 +1,16 @@ +Deimos.configure do + # Define a consumer + consumer do + class_name 'WidgetConsumer' + topic 'TopicToConsume' + schema 'Widget' + namespace 'com.my-namespace' + key_config schema: :MyKeySchema-key + # include Phobos / RubyKafka configs + start_from_beginning true + heartbeat_interval 10 + use_schema_classes true + end + +end + diff --git a/spec/snapshots/consumer_generator_new_deimos_config.snap b/spec/snapshots/consumer_generator_new_deimos_config.snap index 37655f09..c132e405 100644 --- a/spec/snapshots/consumer_generator_new_deimos_config.snap +++ b/spec/snapshots/consumer_generator_new_deimos_config.snap @@ -5,7 +5,7 @@ Deimos.configure do topic 'TopicToConsume' schema 'Widget' namespace 'com.my-namespace' - key_config field: :id + key_config field: :a-string # include Phobos / RubyKafka configs start_from_beginning true heartbeat_interval 10 From ce5b3406d429b1e15427f36c3adc8c72719abfb4 Mon Sep 17 00:00:00 2001 From: jenchanflipp <72824964+jenchanflipp@users.noreply.github.com> Date: Thu, 26 Jan 2023 09:50:55 -0500 Subject: [PATCH 7/7] GUILD-690: Deleting unused snapshots --- spec/snapshots/consumer_generator_config.snap | 16 ---------------- ...onsumer_generator_config_with_config_arg.snap | 15 --------------- 2 files changed, 31 deletions(-) delete mode 100644 spec/snapshots/consumer_generator_config.snap delete mode 100644 spec/snapshots/consumer_generator_config_with_config_arg.snap diff --git a/spec/snapshots/consumer_generator_config.snap b/spec/snapshots/consumer_generator_config.snap deleted file mode 100644 index 37655f09..00000000 --- a/spec/snapshots/consumer_generator_config.snap +++ /dev/null @@ -1,16 +0,0 @@ -Deimos.configure do - # Define a consumer - consumer do - class_name 'WidgetConsumer' - topic 'TopicToConsume' - schema 'Widget' - namespace 'com.my-namespace' - key_config field: :id - # include Phobos / RubyKafka configs - start_from_beginning true - heartbeat_interval 10 - use_schema_classes true - end - -end - diff --git a/spec/snapshots/consumer_generator_config_with_config_arg.snap b/spec/snapshots/consumer_generator_config_with_config_arg.snap deleted file mode 100644 index c2cc2014..00000000 --- a/spec/snapshots/consumer_generator_config_with_config_arg.snap +++ /dev/null @@ -1,15 +0,0 @@ -Deimos.configure do - # Define a consumer - consumer do - class_name 'WidgetConsumer' - topic 'TopicToConsume' - schema 'Widget' - namespace 'com.my-namespace' - key_config field: :id - # include Phobos / RubyKafka configs - start_from_beginning true - heartbeat_interval 10 - use_schema_classes true - end - -end \ No newline at end of file