Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/ruby.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,5 @@ jobs:
run: bundle exec rake test
- name: Run liquid-spec
run: bundle exec rake run
- name: Check feature tag coverage
run: bundle exec rake coverage_check
17 changes: 8 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -90,10 +90,9 @@ LiquidSpec.setup do
require "my_liquid"
end

# Declare which features you support
# Declare what your adapter can't handle (default: run everything)
LiquidSpec.configure do |config|
config.features = [:core] # enables liquid_ruby suite
# Add :shopify_tags, :shopify_objects, :shopify_filters for Shopify themes
config.missing_features = [:shopify_tags, :shopify_filters]
end

# Parse template source into a template object
Expand Down Expand Up @@ -146,15 +145,15 @@ Each spec includes a detailed `hint` explaining how the feature should be implem

### Feature-Based Suite Selection

Suites run based on feature declarations:
Suites run by default. Declare what your adapter can't handle to skip specific specs:

```ruby
LiquidSpec.configure do |config|
# Just core Liquid (liquid_ruby + shopify_production_recordings)
config.features = [:core]
# Full Shopify theme support (adds shopify_theme_dawn)
config.features = [:core, :shopify_tags, :shopify_objects, :shopify_filters]
# Run everything (default — empty denylist)
config.missing_features = []

# Skip Shopify-specific specs (for adapters without Shopify extensions)
config.missing_features = [:shopify_tags, :shopify_objects, :shopify_filters]
end
```

Expand Down
54 changes: 54 additions & 0 deletions Rakefile
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,60 @@ task :matrix do
system("bundle", "exec", "ruby", "bin/liquid-spec", "matrix", "--all") || abort
end

desc "Verify every spec feature tag is covered by at least one reference adapter"
task :coverage_check do
require "yaml"
require "set"

base = File.expand_path(__dir__)

# Collect all feature tags used across spec YAML files
spec_tags = Set.new
Dir.glob(File.join(base, "specs/**/*.yml")).each do |f|
begin
data = YAML.safe_load(File.read(f), permitted_classes: [Symbol, Range], aliases: true)
next unless data

meta = data.is_a?(Hash) ? (data["_metadata"] || {}) : {}
(meta["features"] || []).each { |t| spec_tags << t.to_sym }

specs = data.is_a?(Hash) ? (data["specs"] || []) : (data.is_a?(Array) ? data : [])
specs.each do |spec|
(spec["features"] || []).each { |t| spec_tags << t.to_sym } if spec.is_a?(Hash)
end
rescue
end
end

# Extract missing_features from each reference adapter via static analysis.
# We parse the config.missing_features = [...] line rather than loading the
# adapter (which would require all adapter dependencies to be installed).
adapter_files = Dir.glob(File.join(base, "examples/*.rb"))
adapter_missing = {}
adapter_files.each do |path|
source = File.read(path)
if (m = source.match(/config\.missing_features\s*=\s*\[(.*?)\]/m))
symbols = m[1].scan(/:(\w+)/).flatten.map(&:to_sym)
adapter_missing[File.basename(path)] = symbols.to_set
end
end

if adapter_missing.empty?
abort "Coverage check FAILED — no reference adapters found in examples/"
end

# Check: every tag must be NOT-missing in at least one adapter
orphans = spec_tags.reject do |tag|
adapter_missing.any? { |_, missing| !missing.include?(tag) }
end

if orphans.any?
abort "Coverage check FAILED — no reference adapter covers these tags:\n #{orphans.sort.join(', ')}\n\nAdd an example adapter that does not include these in missing_features."
else
puts "Coverage check passed: all #{spec_tags.size} feature tags covered across #{adapter_missing.size} reference adapters."
end
end

# Spec generation tasks (only needed for development)
import("tasks/liquid_ruby.rake")
import("tasks/standard_filters.rake")
Expand Down
2 changes: 1 addition & 1 deletion examples/json_rpc_ruby_liquid.rb
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
LiquidSpec.configure do |config|
# JSON-RPC adapters support core features including runtime_drops
# because they implement bidirectional callbacks
config.features = [:core, :strict_parsing]
config.missing_features = [:ruby_types, :ruby_drops, :binary_data, :lax_parsing, :activesupport, :template_factory, :shopify_filters, :shopify_includes, :shopify_blank, :shopify_error_handling, :shopify_error_format, :shopify_string_access]
end

LiquidSpec.compile do |ctx, source, parse_options|
Expand Down
2 changes: 1 addition & 1 deletion examples/liquid_c.rb
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def parse_expression(markup, safe: false)
end

LiquidSpec.configure do |config|
config.features = [:core, :lax_parsing]
config.missing_features = [:runtime_drops, :ruby_types, :ruby_drops, :binary_data, :activesupport, :template_factory, :shopify_filters, :shopify_includes, :shopify_blank, :shopify_error_handling, :shopify_error_format, :shopify_string_access]
end

LiquidSpec.compile do |ctx, source, parse_options|
Expand Down
2 changes: 1 addition & 1 deletion examples/liquid_c_strict.rb
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
end

LiquidSpec.configure do |config|
config.features = [:core, :strict_parsing]
config.missing_features = [:runtime_drops, :ruby_types, :ruby_drops, :binary_data, :lax_parsing, :activesupport, :template_factory, :shopify_filters, :shopify_includes, :shopify_blank, :shopify_error_handling, :shopify_error_format, :shopify_string_access]
end

LiquidSpec.compile do |ctx, source, parse_options|
Expand Down
2 changes: 1 addition & 1 deletion examples/liquid_ruby.rb
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
end

LiquidSpec.configure do |config|
config.features = [:core, :strict_parsing, :strict2_parsing, :ruby_types]
config.missing_features = [:shopify_filters, :shopify_includes, :shopify_blank, :shopify_error_handling, :shopify_error_format, :shopify_string_access, :activesupport, :lax_parsing]
end

LiquidSpec.compile do |ctx, source, parse_options|
Expand Down
2 changes: 1 addition & 1 deletion examples/liquid_ruby_activesupport.rb
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
end

LiquidSpec.configure do |config|
config.features = [:core, :activesupport, :strict_parsing]
config.missing_features = [:shopify_filters, :shopify_includes, :shopify_blank, :shopify_error_handling, :shopify_error_format, :shopify_string_access, :lax_parsing]
end

LiquidSpec.compile do |ctx, source, parse_options|
Expand Down
2 changes: 1 addition & 1 deletion examples/liquid_ruby_lax.rb
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
end

LiquidSpec.configure do |config|
config.features = [:core, :lax_parsing]
config.missing_features = [:shopify_filters, :shopify_includes, :shopify_blank, :shopify_error_handling, :shopify_error_format, :shopify_string_access, :activesupport]
end

LiquidSpec.compile do |ctx, source, parse_options|
Expand Down
2 changes: 1 addition & 1 deletion examples/liquid_ruby_lax_activesupport.rb
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
end

LiquidSpec.configure do |config|
config.features = [:core, :lax_parsing, :activesupport]
config.missing_features = [:shopify_filters, :shopify_includes, :shopify_blank, :shopify_error_handling, :shopify_error_format, :shopify_string_access]
end

LiquidSpec.compile do |ctx, source, parse_options|
Expand Down
2 changes: 1 addition & 1 deletion examples/liquid_ruby_shopify.rb
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ def render_to_output_buffer(context, output)
end

LiquidSpec.configure do |config|
config.features = [:core, :strict_parsing, :ruby_types, :shopify_filters]
config.missing_features = [:lax_parsing]
config.suites = [:benchmarks]
config.filter = "shopify_"
end
Expand Down
2 changes: 1 addition & 1 deletion examples/liquid_ruby_yjit.rb
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
end

LiquidSpec.configure do |config|
config.features = [:core, :strict_parsing, :strict2_parsing, :ruby_types]
config.missing_features = [:shopify_filters, :shopify_includes, :shopify_blank, :shopify_error_handling, :shopify_error_format, :shopify_string_access, :activesupport, :lax_parsing]
end

LiquidSpec.compile do |ctx, source, parse_options|
Expand Down
2 changes: 1 addition & 1 deletion examples/liquid_ruby_zjit.rb
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
end

LiquidSpec.configure do |config|
config.features = [:core, :strict_parsing, :strict2_parsing, :ruby_types]
config.missing_features = [:shopify_filters, :shopify_includes, :shopify_blank, :shopify_error_handling, :shopify_error_format, :shopify_string_access, :activesupport, :lax_parsing]
end

LiquidSpec.compile do |ctx, source, parse_options|
Expand Down
19 changes: 9 additions & 10 deletions lib/liquid/spec/adapter_runner.rb
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ class AdapterRunner
TEST_TIME = Time.utc(2024, 1, 1, 0, 1, 58).freeze
TEST_TZ = "America/New_York"

attr_reader :name, :features, :ctx
attr_reader :name, :missing_features, :ctx

def initialize(name: nil)
@name = name
@features = Set.new([:core])
@missing_features = Set.new
@setup_block = nil
@compile_block = nil
@render_block = nil
Expand Down Expand Up @@ -43,8 +43,8 @@ def load_dsl(path)
@render_block = ::LiquidSpec.instance_variable_get(:@render_block)

config = ::LiquidSpec.instance_variable_get(:@config)
if config&.respond_to?(:features)
set_features(config.features)
if config&.respond_to?(:missing_features)
set_missing_features(config.missing_features)
end

# Validate block signatures
Expand Down Expand Up @@ -145,15 +145,14 @@ def on_render(&block)
@render_block = block
end

# Set features
def set_features(features)
@features = Set.new(features.map(&:to_sym))
@features << :core unless @features.include?(:core)
# Set missing features
def set_missing_features(missing_features)
@missing_features = Set.new(missing_features.map(&:to_sym))
end

# Check if adapter can run a spec
def can_run?(spec)
spec.runnable_with?(@features)
!spec.skipped_by?(@missing_features)
end

# Run a batch of specs
Expand Down Expand Up @@ -189,7 +188,7 @@ def run_single(spec)
return SpecResult.new(
spec: spec,
status: :skipped,
reason: "Missing features: #{(spec.required_features - @features.to_a).join(", ")}",
reason: "Adapter does not support: #{(spec.features & @missing_features.to_a).join(", ")}",
)
end

Expand Down
40 changes: 9 additions & 31 deletions lib/liquid/spec/cli/adapter_dsl.rb
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
#
# LiquidSpec.configure do |config|
# config.suite = :liquid_ruby
# config.features = [:core, :lax_parsing]
# config.missing_features = [:shopify_tags]
# end
#
# LiquidSpec.compile do |ctx, source, parse_options|
Expand Down Expand Up @@ -67,57 +67,35 @@ def format_timeout(value)
shopify_error_handling: "Shopify-specific error handling and recovery behavior",
}.freeze

# Feature expansions - declaring a feature automatically includes these
# :core is the "full implementation" feature that includes runtime drop support
# JSON-RPC adapters that can't support runtime drops should not declare :core
FEATURE_EXPANSIONS = {
core: [:runtime_drops, :inline_errors],
}.freeze

# Default features when no config is set (matches Configuration defaults after expansion)
DEFAULT_FEATURES = [:core, :runtime_drops, :inline_errors].freeze

class Configuration
attr_accessor :suite, :filter, :verbose, :strict_only
attr_reader :features, :known_failures, :suites
attr_reader :missing_features, :known_failures, :suites

def initialize
@suite = :all
@suites = nil
@filter = nil
@verbose = false
@strict_only = false
@features = [:core]
@missing_features = []
@known_failures = []
expand_features!
end

# Set multiple suites to run (overrides suite)
def suites=(list)
@suites = Array(list).map(&:to_sym)
end

def features=(list)
@features = Array(list).map(&:to_sym)
expand_features!
def missing_features=(list)
@missing_features = Array(list).map(&:to_sym)
end

def known_failures=(list)
@known_failures = Array(list).map(&:to_s)
end

def feature?(name)
@features.include?(name.to_sym)
end

private

def expand_features!
FEATURE_EXPANSIONS.each do |feature, includes|
if @features.include?(feature)
@features |= includes
end
end
def missing_feature?(name)
@missing_features.include?(name.to_sym)
end
end

Expand All @@ -140,8 +118,8 @@ def configure
@config
end

def features
@config&.instance_variable_get(:@features) || DEFAULT_FEATURES
def missing_features
@config&.instance_variable_get(:@missing_features) || []
end

# Declare Ruby flags this adapter needs (e.g. "--yjit").
Expand Down
14 changes: 7 additions & 7 deletions lib/liquid/spec/cli/features.rb
Original file line number Diff line number Diff line change
Expand Up @@ -117,10 +117,10 @@ def self.run(args)
puts "liquid-spec Features"
puts "=" * 60
puts
puts "Features control which specs are run. Declare them in your adapter:"
puts "Features are tags on specs. Declare what your adapter can't run:"
puts
puts " LiquidSpec.configure do |config|"
puts " config.features = [:core, :strict_parsing]"
puts " config.missing_features = [:shopify_tags, :shopify_filters]"
puts " end"
puts
puts "-" * 60
Expand All @@ -147,8 +147,8 @@ def self.run(args)
puts " #{feature}: #{count}"
end
puts
puts "Recommended starter config:"
puts " config.features = [:core, :strict_parsing]"
puts "Adapter with no limitations (runs all specs):"
puts " config.missing_features = []"
puts
end

Expand All @@ -167,9 +167,9 @@ def self.count_specs_by_feature
specs.each do |spec|
next unless spec.is_a?(Hash)

# Get required features from spec or metadata
features = spec["required_features"] ||
(metadata && metadata["required_features"]) ||
# Get features from spec or metadata
features = spec["features"] ||
(metadata && metadata["features"]) ||
[]

features = [features] unless features.is_a?(Array)
Expand Down
Loading
Loading