From e5aeb87161c1c9ffeac769201a80eb2f0afc3742 Mon Sep 17 00:00:00 2001 From: Sutou Kouhei Date: Mon, 5 Feb 2024 15:28:53 +0900 Subject: [PATCH] Add benchmark mode It uses Google Benchmark compatible JSON output. --- lib/grntest/base-result.rb | 22 +++- lib/grntest/execution-context.rb | 4 +- lib/grntest/executors/base-executor.rb | 28 +++- lib/grntest/reporters.rb | 17 ++- lib/grntest/reporters/base-reporter.rb | 10 +- .../reporters/benchmark-json-reporter.rb | 121 ++++++++++++++++++ lib/grntest/test-runner.rb | 21 ++- lib/grntest/tester.rb | 11 +- 8 files changed, 209 insertions(+), 25 deletions(-) create mode 100644 lib/grntest/reporters/benchmark-json-reporter.rb diff --git a/lib/grntest/base-result.rb b/lib/grntest/base-result.rb index 4db45d3..39f8465 100644 --- a/lib/grntest/base-result.rb +++ b/lib/grntest/base-result.rb @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- -# -# Copyright (C) 2012-2013 Kouhei Sutou +# Copyright (C) 2012-2024 Sutou Kouhei # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,16 +15,26 @@ module Grntest class BaseResult - attr_accessor :elapsed_time + attr_accessor :cpu_elapsed_time + attr_accessor :real_elapsed_time def initialize - @elapsed_time = 0 + @cpu_elapsed_time = 0 + @real_elapsed_time = 0 end def measure - start_time = Time.now + cpu_start_time = Process.times + real_start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) yield ensure - @elapsed_time = Time.now - start_time + cpu_finish_time = Process.times + real_finish_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @cpu_elapsed_time = + (cpu_finish_time.utime - cpu_start_time.utime) + + (cpu_finish_time.stime - cpu_start_time.stime) + + (cpu_finish_time.cutime - cpu_start_time.cutime) + + (cpu_finish_time.cstime - cpu_start_time.cstime) + @real_elapsed_time = real_finish_time - real_start_time end end end diff --git a/lib/grntest/execution-context.rb b/lib/grntest/execution-context.rb index eddf14c..2da14c4 100644 --- a/lib/grntest/execution-context.rb +++ b/lib/grntest/execution-context.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2021 Sutou Kouhei +# Copyright (C) 2012-2024 Sutou Kouhei # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -39,6 +39,7 @@ class ExecutionContext attr_writer :collect_query_log attr_writer :debug attr_accessor :platform + attr_accessor :benchmarks def initialize @logging = true @base_directory = Pathname(".") @@ -70,6 +71,7 @@ def initialize @collect_query_log = false @debug = false @platform = guess_platform + @benchmarks = [] end def logging? diff --git a/lib/grntest/executors/base-executor.rb b/lib/grntest/executors/base-executor.rb index eb7e2ba..4839f1e 100644 --- a/lib/grntest/executors/base-executor.rb +++ b/lib/grntest/executors/base-executor.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2023 Sutou Kouhei +# Copyright (C) 2012-2024 Sutou Kouhei # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -48,6 +48,8 @@ def initialize(context) @raw_status_response = nil @features = nil @substitutions = {} + @noop_benchmark_result = BenchmarkResult.new("noop", 1, 1) + @benchmark_result = @noop_benchmark_result end def execute(script_path) @@ -520,6 +522,18 @@ def execute_directive_remove_substitution(line, content, options) @substitutions.delete(pattern) end + def execute_directive_start_benchmark(line, content, options) + _, n_items, n_iterations, name = content.split(" ", 4) + n_items = Integer(n_items, 10) + n_iterations = Integer(n_iterations, 10) + @benchmark_result = BenchmarkResult.new(name, n_items, n_iterations) + @context.benchmarks << @benchmark_result + end + + def execute_directive_finish_benchmark(line, content, options) + @benchmark_result = @noop_benchmark_result + end + def execute_directive(parser, line, content) command, *options = Shellwords.split(content) case command @@ -579,6 +593,10 @@ def execute_directive(parser, line, content) execute_directive_add_substitution(line, content, options) when "remove-substitution" execute_directive_remove_substitution(line, content, options) + when "start-benchmark" + execute_directive_start_benchmark(line, content, options) + when "finish-benchmark" + execute_directive_finish_benchmark(line, content, options) else log_input(line) log_error("#|e| unknown directive: <#{command}>") @@ -628,8 +646,12 @@ def execute_command(command) timeout = @context.timeout response = nil begin - Timeout.timeout(timeout) do - response = send_command(command) + @benchmark_result.measure do + @benchmark_result.n_iterations.times do + Timeout.timeout(timeout) do + response = send_command(command) + end + end end rescue Timeout::Error log_error("# error: timeout (#{timeout}s)") diff --git a/lib/grntest/reporters.rb b/lib/grntest/reporters.rb index 4da1cf9..cf39a8c 100644 --- a/lib/grntest/reporters.rb +++ b/lib/grntest/reporters.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2020 Sutou Kouhei +# Copyright (C) 2012-2024 Sutou Kouhei # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -13,27 +13,30 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -require "grntest/reporters/mark-reporter" +require "grntest/reporters/benchmark-json-reporter" require "grntest/reporters/buffered-mark-reporter" -require "grntest/reporters/stream-reporter" require "grntest/reporters/inplace-reporter" +require "grntest/reporters/mark-reporter" require "grntest/reporters/progress-reporter" +require "grntest/reporters/stream-reporter" module Grntest module Reporters class << self def create_reporter(tester) case tester.reporter - when :mark - MarkReporter.new(tester) + when :"benchmark-json" + BenchmarkJSONReporter.new(tester) when :"buffered-mark" BufferedMarkReporter.new(tester) - when :stream - StreamReporter.new(tester) when :inplace InplaceReporter.new(tester) + when :mark + MarkReporter.new(tester) when :progress ProgressReporter.new(tester) + when :stream + StreamReporter.new(tester) end end end diff --git a/lib/grntest/reporters/base-reporter.rb b/lib/grntest/reporters/base-reporter.rb index c87817d..c39ff11 100644 --- a/lib/grntest/reporters/base-reporter.rb +++ b/lib/grntest/reporters/base-reporter.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2023 Sutou Kouhei +# Copyright (C) 2012-2024 Sutou Kouhei # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -45,7 +45,7 @@ def report_summary(result) puts(statistics_header) puts(colorize(statistics(result), result)) pass_ratio = result.pass_ratio - elapsed_time = result.elapsed_time + elapsed_time = result.real_elapsed_time summary = "%.4g%% passed in %.4fs." % [pass_ratio, elapsed_time] puts(colorize(summary, result)) end @@ -78,10 +78,10 @@ def statistics(result) end def throughput(result) - if result.elapsed_time.zero? + if result.real_elapsed_time.zero? tests_per_second = 0 else - tests_per_second = result.n_tests / result.elapsed_time + tests_per_second = result.n_tests / result.real_elapsed_time end tests_per_second end @@ -160,7 +160,7 @@ def report_test_result(result, label) end def test_result_message(result, label) - elapsed_time = result.elapsed_time + elapsed_time = result.real_elapsed_time formatted_elapsed_time = "%.4fs" % elapsed_time formatted_elapsed_time = colorize(formatted_elapsed_time, elapsed_time_status(elapsed_time)) diff --git a/lib/grntest/reporters/benchmark-json-reporter.rb b/lib/grntest/reporters/benchmark-json-reporter.rb new file mode 100644 index 0000000..04c54d3 --- /dev/null +++ b/lib/grntest/reporters/benchmark-json-reporter.rb @@ -0,0 +1,121 @@ +# Copyright (C) 2024 Sutou Kouhei +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +require "json" +require "time" + +require "grntest/reporters/base-reporter" + +module Grntest + module Reporters + class BenchmarkJSONReporter < BaseReporter + def initialize(tester) + super + end + + def on_start(result) + puts(<<-JSON) +{ + "context": {" + "date": #{Time.now.iso8601.to_json}, + "host_name": #{Socket.gethostname.to_json}, + "executable": #{@tester.testee.to_json}, + "num_cpus": #{Etc.nprocessors}, + JSON + cpu_cycles_per_second = detect_cpu_cycles_per_second + if cpu_cycles_per_second + puts(<<-JSON) + "mhz_per_cpu": #{cpu_cycles_per_second / 1_000_000.0}, + JSON + end + puts(<<-JSON) + "caches": [] + }, + "benchmarks": [ + JSON + end + + def on_worker_start(worker) + end + + def on_suite_start(worker) + end + + def on_test_start(worker) + end + + def on_test_success(worker, result) + end + + def on_test_failure(worker, result) + end + + def on_test_leak(worker, result) + end + + def on_test_omission(worker, result) + end + + def on_test_no_check(worker, result) + end + + def on_test_finish(worker, result) + benchmarks = result.benchmarks.collect do |benchmark| + <<-JSON.chomp + { + "name": #{result.test_name.to_json}, + "run_name": #{benchmark.name.to_json}, + "run_type": "iteration", + "iterations": #{benchmark.n_iterations}, + "real_time": #{benchmark.real_elapsed_time}, + "cpu_time": #{benchmark.cpu_elapsed_time}, + "time_unit": "s", + "items_per_second": #{benchmark.items_per_second}, + } + JSON + end + puts(benchmarks.join(",\n")) + end + + def on_suite_finish(worker) + end + + def on_worker_finish(worker) + end + + def on_finish(result) + puts(<<-JSON) + ] +} + JSON + end + + private + def detect_cpu_cycles_per_second + if File.exist?("/proc/cpuinfo") + File.open("/proc/cpuinfo") do |cpuinfo| + cpuinfo.each_line do |line| + case line + when /\Acpu MHz\s+: ([\d.]+)/ + return Float($1) + end + end + end + end + nil + end + end + end +end diff --git a/lib/grntest/test-runner.rb b/lib/grntest/test-runner.rb index e9a7f36..fa67724 100644 --- a/lib/grntest/test-runner.rb +++ b/lib/grntest/test-runner.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2022 Sutou Kouhei +# Copyright (C) 2012-2024 Sutou Kouhei # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -28,10 +28,27 @@ require "grntest/base-result" module Grntest + class BenchmarkResult < BaseResult + attr_reader :name + attr_reader :n_items + attr_reader :n_iterations + def initialize(name, n_items, n_iterations) + super() + @name = name + @n_items = n_items + @n_iterations = n_iterations + end + + def items_per_second + @n_items / @real_elapsed_time + end + end + class TestResult < BaseResult attr_accessor :worker_id, :test_name attr_accessor :expected, :actual, :n_leaked_objects attr_writer :omitted + attr_accessor :benchmarks def initialize(worker) super() @worker_id = worker.id @@ -40,6 +57,7 @@ def initialize(worker) @expected = nil @n_leaked_objects = 0 @omitted = false + @benchmarks = [] end def status @@ -159,6 +177,7 @@ def execute_groonga_script(result) check_memory_leak(context) result.omitted = context.omitted? result.actual = context.result + result.benchmarks = context.benchmarks context.close_logs end end diff --git a/lib/grntest/tester.rb b/lib/grntest/tester.rb index 9a55a11..c22cd4c 100644 --- a/lib/grntest/tester.rb +++ b/lib/grntest/tester.rb @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2023 Sutou Kouhei +# Copyright (C) 2012-2024 Sutou Kouhei # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -161,6 +161,7 @@ def create_option_parser(tester, tag) :stream, :inplace, :progress, + :"benchmark-json", ] available_reporter_labels = available_reporters.join(", ") parser.on("--reporter=REPORTER", available_reporters, @@ -299,6 +300,14 @@ def create_option_parser(tester, tag) srand(seed) end + parser.on("--[no-]benchmark", + "Set options for benchmark") do |benchmark| + if benchmark + tester.n_workers = 1 + tester.reporter = :"benchmark-json" + end + end + parser.on("--version", "Show version and exit") do puts(VERSION)