Add benchmark mode

It uses Google Benchmark compatible JSON output.

lib/grntest/base-result.rb

@@ -1,6 +1,4 @@
-# -*- coding: utf-8 -*-
-#
-# Copyright (C) 2012-2013  Kouhei Sutou <[email protected]>
+# Copyright (C) 2012-2024  Sutou Kouhei <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,16 +15,26 @@
 
 module Grntest
   class BaseResult
-    attr_accessor :elapsed_time
+    attr_accessor :cpu_elapsed_time
+    attr_accessor :real_elapsed_time
     def initialize
-      @elapsed_time = 0
+      @cpu_elapsed_time = 0
+      @real_elapsed_time = 0
     end
 
     def measure
-      start_time = Time.now
+      cpu_start_time = Process.times
+      real_start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       yield
     ensure
-      @elapsed_time = Time.now - start_time
+      cpu_finish_time = Process.times
+      real_finish_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      @cpu_elapsed_time =
+        (cpu_finish_time.utime - cpu_start_time.utime) +
+        (cpu_finish_time.stime - cpu_start_time.stime) +
+        (cpu_finish_time.cutime - cpu_start_time.cutime) +
+        (cpu_finish_time.cstime - cpu_start_time.cstime)
+      @real_elapsed_time = real_finish_time - real_start_time
     end
   end
 end

lib/grntest/execution-context.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2021  Sutou Kouhei <[email protected]>
+# Copyright (C) 2012-2024  Sutou Kouhei <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -39,6 +39,7 @@ class ExecutionContext
     attr_writer :collect_query_log
     attr_writer :debug
     attr_accessor :platform
+    attr_accessor :benchmarks
     def initialize
       @logging = true
       @base_directory = Pathname(".")
@@ -70,6 +71,7 @@ def initialize
       @collect_query_log = false
       @debug = false
       @platform = guess_platform
+      @benchmarks = []
     end
 
     def logging?

lib/grntest/executors/base-executor.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2023  Sutou Kouhei <[email protected]>
+# Copyright (C) 2012-2024  Sutou Kouhei <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -48,6 +48,8 @@ def initialize(context)
         @raw_status_response = nil
         @features = nil
         @substitutions = {}
+        @noop_benchmark_result = BenchmarkResult.new("noop", 1, 1)
+        @benchmark_result = @noop_benchmark_result
       end
 
       def execute(script_path)
@@ -520,6 +522,18 @@ def execute_directive_remove_substitution(line, content, options)
         @substitutions.delete(pattern)
       end
 
+      def execute_directive_start_benchmark(line, content, options)
+        _, n_items, n_iterations, name = content.split(" ", 4)
+        n_items = Integer(n_items, 10)
+        n_iterations = Integer(n_iterations, 10)
+        @benchmark_result = BenchmarkResult.new(name, n_items, n_iterations)
+        @context.benchmarks << @benchmark_result
+      end
+
+      def execute_directive_finish_benchmark(line, content, options)
+        @benchmark_result = @noop_benchmark_result
+      end
+
       def execute_directive(parser, line, content)
         command, *options = Shellwords.split(content)
         case command
@@ -579,6 +593,10 @@ def execute_directive(parser, line, content)
           execute_directive_add_substitution(line, content, options)
         when "remove-substitution"
           execute_directive_remove_substitution(line, content, options)
+        when "start-benchmark"
+          execute_directive_start_benchmark(line, content, options)
+        when "finish-benchmark"
+          execute_directive_finish_benchmark(line, content, options)
         else
           log_input(line)
           log_error("#|e| unknown directive: <#{command}>")
@@ -628,8 +646,12 @@ def execute_command(command)
         timeout = @context.timeout
         response = nil
         begin
-          Timeout.timeout(timeout) do
-            response = send_command(command)
+          @benchmark_result.measure do
+            @benchmark_result.n_iterations.times do
+              Timeout.timeout(timeout) do
+                response = send_command(command)
+              end
+            end
           end
         rescue Timeout::Error
           log_error("# error: timeout (#{timeout}s)")

lib/grntest/reporters.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2020  Sutou Kouhei <[email protected]>
+# Copyright (C) 2012-2024  Sutou Kouhei <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -13,27 +13,30 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-require "grntest/reporters/mark-reporter"
+require "grntest/reporters/benchmark-json-reporter"
 require "grntest/reporters/buffered-mark-reporter"
-require "grntest/reporters/stream-reporter"
 require "grntest/reporters/inplace-reporter"
+require "grntest/reporters/mark-reporter"
 require "grntest/reporters/progress-reporter"
+require "grntest/reporters/stream-reporter"
 
 module Grntest
   module Reporters
     class << self
       def create_reporter(tester)
         case tester.reporter
-        when :mark
-          MarkReporter.new(tester)
+        when :"benchmark-json"
+          BenchmarkJSONReporter.new(tester)
         when :"buffered-mark"
           BufferedMarkReporter.new(tester)
-        when :stream
-          StreamReporter.new(tester)
         when :inplace
           InplaceReporter.new(tester)
+        when :mark
+          MarkReporter.new(tester)
         when :progress
           ProgressReporter.new(tester)
+        when :stream
+          StreamReporter.new(tester)
         end
       end
     end

lib/grntest/reporters/base-reporter.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2023  Sutou Kouhei <[email protected]>
+# Copyright (C) 2012-2024  Sutou Kouhei <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -45,7 +45,7 @@ def report_summary(result)
         puts(statistics_header)
         puts(colorize(statistics(result), result))
         pass_ratio = result.pass_ratio
-        elapsed_time = result.elapsed_time
+        elapsed_time = result.real_elapsed_time
         summary = "%.4g%% passed in %.4fs." % [pass_ratio, elapsed_time]
         puts(colorize(summary, result))
       end
@@ -78,10 +78,10 @@ def statistics(result)
       end
 
       def throughput(result)
-        if result.elapsed_time.zero?
+        if result.real_elapsed_time.zero?
           tests_per_second = 0
         else
-          tests_per_second = result.n_tests / result.elapsed_time
+          tests_per_second = result.n_tests / result.real_elapsed_time
         end
         tests_per_second
       end
@@ -160,7 +160,7 @@ def report_test_result(result, label)
       end
 
       def test_result_message(result, label)
-        elapsed_time = result.elapsed_time
+        elapsed_time = result.real_elapsed_time
         formatted_elapsed_time = "%.4fs" % elapsed_time
         formatted_elapsed_time = colorize(formatted_elapsed_time,
                                           elapsed_time_status(elapsed_time))

lib/grntest/reporters/benchmark-json-reporter.rb

@@ -0,0 +1,121 @@
+# Copyright (C) 2024  Sutou Kouhei <[email protected]>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+require "json"
+require "time"
+
+require "grntest/reporters/base-reporter"
+
+module Grntest
+  module Reporters
+    class BenchmarkJSONReporter < BaseReporter
+      def initialize(tester)
+        super
+      end
+
+      def on_start(result)
+        puts(<<-JSON)
+{
+  "context": {"
+    "date": #{Time.now.iso8601.to_json},
+    "host_name": #{Socket.gethostname.to_json},
+    "executable": #{@tester.testee.to_json},
+    "num_cpus": #{Etc.nprocessors},
+        JSON
+        cpu_cycles_per_second = detect_cpu_cycles_per_second
+        if cpu_cycles_per_second
+          puts(<<-JSON)
+    "mhz_per_cpu": #{cpu_cycles_per_second / 1_000_000.0},
+          JSON
+        end
+        puts(<<-JSON)
+    "caches": []
+  },
+  "benchmarks": [
+        JSON
+      end
+
+      def on_worker_start(worker)
+      end
+
+      def on_suite_start(worker)
+      end
+
+      def on_test_start(worker)
+      end
+
+      def on_test_success(worker, result)
+      end
+
+      def on_test_failure(worker, result)
+      end
+
+      def on_test_leak(worker, result)
+      end
+
+      def on_test_omission(worker, result)
+      end
+
+      def on_test_no_check(worker, result)
+      end
+
+      def on_test_finish(worker, result)
+        benchmarks = result.benchmarks.collect do |benchmark|
+          <<-JSON.chomp
+    {
+      "name": #{result.test_name.to_json},
+      "run_name": #{benchmark.name.to_json},
+      "run_type": "iteration",
+      "iterations": #{benchmark.n_iterations},
+      "real_time": #{benchmark.real_elapsed_time},
+      "cpu_time": #{benchmark.cpu_elapsed_time},
+      "time_unit": "s",
+      "items_per_second": #{benchmark.items_per_second},
+    }
+          JSON
+        end
+        puts(benchmarks.join(",\n"))
+      end
+
+      def on_suite_finish(worker)
+      end
+
+      def on_worker_finish(worker)
+      end
+
+      def on_finish(result)
+        puts(<<-JSON)
+  ]
+}
+        JSON
+      end
+
+      private
+      def detect_cpu_cycles_per_second
+        if File.exist?("/proc/cpuinfo")
+          File.open("/proc/cpuinfo") do |cpuinfo|
+            cpuinfo.each_line do |line|
+              case line
+              when /\Acpu MHz\s+: ([\d.]+)/
+                return Float($1)
+              end
+            end
+          end
+        end
+        nil
+      end
+    end
+  end
+end

lib/grntest/test-runner.rb

@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2022  Sutou Kouhei <[email protected]>
+# Copyright (C) 2012-2024  Sutou Kouhei <[email protected]>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -28,10 +28,27 @@
 require "grntest/base-result"
 
 module Grntest
+  class BenchmarkResult < BaseResult
+    attr_reader :name
+    attr_reader :n_items
+    attr_reader :n_iterations
+    def initialize(name, n_items, n_iterations)
+      super()
+      @name = name
+      @n_items = n_items
+      @n_iterations = n_iterations
+    end
+
+    def items_per_second
+      @n_items / @real_elapsed_time
+    end
+  end
+
   class TestResult < BaseResult
     attr_accessor :worker_id, :test_name
     attr_accessor :expected, :actual, :n_leaked_objects
     attr_writer :omitted
+    attr_accessor :benchmarks
     def initialize(worker)
       super()
       @worker_id = worker.id
@@ -40,6 +57,7 @@ def initialize(worker)
       @expected = nil
       @n_leaked_objects = 0
       @omitted = false
+      @benchmarks = []
     end
 
     def status
@@ -159,6 +177,7 @@ def execute_groonga_script(result)
         check_memory_leak(context)
         result.omitted = context.omitted?
         result.actual = context.result
+        result.benchmarks = context.benchmarks
         context.close_logs
       end
     end