diff --git a/README.md b/README.md index 8feff07f..bba0d1ef 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,33 @@ It's also convenient for profiling, debugging, etc, especially since all benchma ruby benchmarks/some_benchmark.rb ``` +### Benchmark organization + +Benchmarks can be organized in three ways: + +1. **Standalone .rb files** - Place a `.rb` file directly in the `benchmarks/` directory: + ``` + benchmarks/fib.rb # Benchmark name: "fib" + ``` + +2. **Single benchmark per directory** - For benchmarks that need additional files (like Gemfiles): + ``` + benchmarks/erubi/ + benchmark.rb # Benchmark name: "erubi" + Gemfile + ``` + +3. **Multiple benchmarks per directory** - For related benchmarks sharing dependencies: + ``` + benchmarks/addressable/ + equality.rb # Benchmark name: "addressable-equality" + join.rb # Benchmark name: "addressable-join" + Gemfile # Shared Gemfile + ``` + + In directories **without** a `benchmark.rb` file, all `.rb` files will be discovered as separate benchmarks. + The benchmark name is derived as `directoryname-suffix` from `suffix.rb` files. + ## Ractor Benchmarks ruby-bench supports Ractor-specific benchmarking with dedicated categories and benchmark directories. diff --git a/benchmarks/addressable-getters/Gemfile b/benchmarks/addressable-getters/Gemfile deleted file mode 100644 index 55401701..00000000 --- a/benchmarks/addressable-getters/Gemfile +++ /dev/null @@ -1,5 +0,0 @@ -# frozen_string_literal: true - -source "https://rubygems.org" - -gem "addressable" diff --git a/benchmarks/addressable-getters/Gemfile.lock b/benchmarks/addressable-getters/Gemfile.lock deleted file mode 100644 index 75a6aa88..00000000 --- a/benchmarks/addressable-getters/Gemfile.lock +++ /dev/null @@ -1,16 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - addressable (2.8.7) - public_suffix (>= 2.0.2, < 7.0) - public_suffix (6.0.2) - -PLATFORMS - aarch64-linux - ruby - -DEPENDENCIES - addressable - -BUNDLED WITH - 2.6.9 diff --git a/benchmarks/addressable-join/Gemfile b/benchmarks/addressable-join/Gemfile deleted file mode 100644 index 55401701..00000000 --- a/benchmarks/addressable-join/Gemfile +++ /dev/null @@ -1,5 +0,0 @@ -# frozen_string_literal: true - -source "https://rubygems.org" - -gem "addressable" diff --git a/benchmarks/addressable-join/Gemfile.lock b/benchmarks/addressable-join/Gemfile.lock deleted file mode 100644 index 75a6aa88..00000000 --- a/benchmarks/addressable-join/Gemfile.lock +++ /dev/null @@ -1,16 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - addressable (2.8.7) - public_suffix (>= 2.0.2, < 7.0) - public_suffix (6.0.2) - -PLATFORMS - aarch64-linux - ruby - -DEPENDENCIES - addressable - -BUNDLED WITH - 2.6.9 diff --git a/benchmarks/addressable-merge/Gemfile b/benchmarks/addressable-merge/Gemfile deleted file mode 100644 index 55401701..00000000 --- a/benchmarks/addressable-merge/Gemfile +++ /dev/null @@ -1,5 +0,0 @@ -# frozen_string_literal: true - -source "https://rubygems.org" - -gem "addressable" diff --git a/benchmarks/addressable-merge/Gemfile.lock b/benchmarks/addressable-merge/Gemfile.lock deleted file mode 100644 index 75a6aa88..00000000 --- a/benchmarks/addressable-merge/Gemfile.lock +++ /dev/null @@ -1,16 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - addressable (2.8.7) - public_suffix (>= 2.0.2, < 7.0) - public_suffix (6.0.2) - -PLATFORMS - aarch64-linux - ruby - -DEPENDENCIES - addressable - -BUNDLED WITH - 2.6.9 diff --git a/benchmarks/addressable-new/Gemfile b/benchmarks/addressable-new/Gemfile deleted file mode 100644 index 55401701..00000000 --- a/benchmarks/addressable-new/Gemfile +++ /dev/null @@ -1,5 +0,0 @@ -# frozen_string_literal: true - -source "https://rubygems.org" - -gem "addressable" diff --git a/benchmarks/addressable-new/Gemfile.lock b/benchmarks/addressable-new/Gemfile.lock deleted file mode 100644 index 75a6aa88..00000000 --- a/benchmarks/addressable-new/Gemfile.lock +++ /dev/null @@ -1,16 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - addressable (2.8.7) - public_suffix (>= 2.0.2, < 7.0) - public_suffix (6.0.2) - -PLATFORMS - aarch64-linux - ruby - -DEPENDENCIES - addressable - -BUNDLED WITH - 2.6.9 diff --git a/benchmarks/addressable-normalize/Gemfile b/benchmarks/addressable-normalize/Gemfile deleted file mode 100644 index 55401701..00000000 --- a/benchmarks/addressable-normalize/Gemfile +++ /dev/null @@ -1,5 +0,0 @@ -# frozen_string_literal: true - -source "https://rubygems.org" - -gem "addressable" diff --git a/benchmarks/addressable-normalize/Gemfile.lock b/benchmarks/addressable-normalize/Gemfile.lock deleted file mode 100644 index 75a6aa88..00000000 --- a/benchmarks/addressable-normalize/Gemfile.lock +++ /dev/null @@ -1,16 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - addressable (2.8.7) - public_suffix (>= 2.0.2, < 7.0) - public_suffix (6.0.2) - -PLATFORMS - aarch64-linux - ruby - -DEPENDENCIES - addressable - -BUNDLED WITH - 2.6.9 diff --git a/benchmarks/addressable-parse/Gemfile b/benchmarks/addressable-parse/Gemfile deleted file mode 100644 index 55401701..00000000 --- a/benchmarks/addressable-parse/Gemfile +++ /dev/null @@ -1,5 +0,0 @@ -# frozen_string_literal: true - -source "https://rubygems.org" - -gem "addressable" diff --git a/benchmarks/addressable-parse/Gemfile.lock b/benchmarks/addressable-parse/Gemfile.lock deleted file mode 100644 index 75a6aa88..00000000 --- a/benchmarks/addressable-parse/Gemfile.lock +++ /dev/null @@ -1,16 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - addressable (2.8.7) - public_suffix (>= 2.0.2, < 7.0) - public_suffix (6.0.2) - -PLATFORMS - aarch64-linux - ruby - -DEPENDENCIES - addressable - -BUNDLED WITH - 2.6.9 diff --git a/benchmarks/addressable-setters/Gemfile b/benchmarks/addressable-setters/Gemfile deleted file mode 100644 index 55401701..00000000 --- a/benchmarks/addressable-setters/Gemfile +++ /dev/null @@ -1,5 +0,0 @@ -# frozen_string_literal: true - -source "https://rubygems.org" - -gem "addressable" diff --git a/benchmarks/addressable-setters/Gemfile.lock b/benchmarks/addressable-setters/Gemfile.lock deleted file mode 100644 index 75a6aa88..00000000 --- a/benchmarks/addressable-setters/Gemfile.lock +++ /dev/null @@ -1,16 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - addressable (2.8.7) - public_suffix (>= 2.0.2, < 7.0) - public_suffix (6.0.2) - -PLATFORMS - aarch64-linux - ruby - -DEPENDENCIES - addressable - -BUNDLED WITH - 2.6.9 diff --git a/benchmarks/addressable-to-s/Gemfile b/benchmarks/addressable-to-s/Gemfile deleted file mode 100644 index 55401701..00000000 --- a/benchmarks/addressable-to-s/Gemfile +++ /dev/null @@ -1,5 +0,0 @@ -# frozen_string_literal: true - -source "https://rubygems.org" - -gem "addressable" diff --git a/benchmarks/addressable-to-s/Gemfile.lock b/benchmarks/addressable-to-s/Gemfile.lock deleted file mode 100644 index 75a6aa88..00000000 --- a/benchmarks/addressable-to-s/Gemfile.lock +++ /dev/null @@ -1,16 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - addressable (2.8.7) - public_suffix (>= 2.0.2, < 7.0) - public_suffix (6.0.2) - -PLATFORMS - aarch64-linux - ruby - -DEPENDENCIES - addressable - -BUNDLED WITH - 2.6.9 diff --git a/benchmarks/addressable-equality/Gemfile b/benchmarks/addressable/Gemfile similarity index 100% rename from benchmarks/addressable-equality/Gemfile rename to benchmarks/addressable/Gemfile diff --git a/benchmarks/addressable-equality/Gemfile.lock b/benchmarks/addressable/Gemfile.lock similarity index 100% rename from benchmarks/addressable-equality/Gemfile.lock rename to benchmarks/addressable/Gemfile.lock diff --git a/benchmarks/addressable-equality/benchmark.rb b/benchmarks/addressable/equality.rb similarity index 100% rename from benchmarks/addressable-equality/benchmark.rb rename to benchmarks/addressable/equality.rb diff --git a/benchmarks/addressable-getters/benchmark.rb b/benchmarks/addressable/getters.rb similarity index 100% rename from benchmarks/addressable-getters/benchmark.rb rename to benchmarks/addressable/getters.rb diff --git a/benchmarks/addressable-join/benchmark.rb b/benchmarks/addressable/join.rb similarity index 100% rename from benchmarks/addressable-join/benchmark.rb rename to benchmarks/addressable/join.rb diff --git a/benchmarks/addressable-merge/benchmark.rb b/benchmarks/addressable/merge.rb similarity index 100% rename from benchmarks/addressable-merge/benchmark.rb rename to benchmarks/addressable/merge.rb diff --git a/benchmarks/addressable-new/benchmark.rb b/benchmarks/addressable/new.rb similarity index 100% rename from benchmarks/addressable-new/benchmark.rb rename to benchmarks/addressable/new.rb diff --git a/benchmarks/addressable-normalize/benchmark.rb b/benchmarks/addressable/normalize.rb similarity index 100% rename from benchmarks/addressable-normalize/benchmark.rb rename to benchmarks/addressable/normalize.rb diff --git a/benchmarks/addressable-parse/benchmark.rb b/benchmarks/addressable/parse.rb similarity index 100% rename from benchmarks/addressable-parse/benchmark.rb rename to benchmarks/addressable/parse.rb diff --git a/benchmarks/addressable-setters/benchmark.rb b/benchmarks/addressable/setters.rb similarity index 100% rename from benchmarks/addressable-setters/benchmark.rb rename to benchmarks/addressable/setters.rb diff --git a/benchmarks/addressable-to-s/benchmark.rb b/benchmarks/addressable/to-s.rb similarity index 100% rename from benchmarks/addressable-to-s/benchmark.rb rename to benchmarks/addressable/to-s.rb diff --git a/lib/benchmark_discovery.rb b/lib/benchmark_discovery.rb new file mode 100644 index 00000000..efb53ea0 --- /dev/null +++ b/lib/benchmark_discovery.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +# BenchmarkDiscovery handles discovering benchmarks in different organization patterns: +# 1. Standalone .rb files in benchmarks/ directory +# 2. Single benchmark.rb in a subdirectory +# 3. Multiple .rb files in a subdirectory (named as directory-suffix, excluding benchmark.rb) +class BenchmarkDiscovery + # Represents a discovered benchmark + BenchmarkEntry = Struct.new(:name, :script_path, :directory, keyword_init: true) + + attr_reader :base_dir + + def initialize(base_dir) + @base_dir = base_dir + end + + # Returns an array of BenchmarkEntry objects + def discover + return [] unless Dir.exist?(base_dir) + + entries = [] + + Dir.children(base_dir).sort.each do |entry| + entry_path = File.join(base_dir, entry) + + if File.file?(entry_path) && entry.end_with?('.rb') + # Pattern 1: Standalone .rb file + entries << BenchmarkEntry.new( + name: entry.delete_suffix('.rb'), + script_path: entry_path, + directory: nil + ) + elsif File.directory?(entry_path) + # Check for patterns 2 and 3 + entries.concat(discover_directory_benchmarks(entry, entry_path)) + end + end + + entries + end + + private + + def discover_directory_benchmarks(dir_name, dir_path) + benchmark_files = find_benchmark_files(dir_path) + return [] if benchmark_files.empty? + + entries = benchmark_files.map do |file| + create_benchmark_entry_in_directory(dir_name, dir_path, file) + end + + entries.sort_by(&:name) + end + + def find_benchmark_files(dir_path) + all_rb_files = Dir.children(dir_path).select { |file| file.end_with?('.rb') } + + # If benchmark.rb exists, only use that (Pattern 2) + if all_rb_files.include?('benchmark.rb') + ['benchmark.rb'] + else + # Otherwise, use all .rb files (Pattern 3) + all_rb_files + end + end + + def create_benchmark_entry_in_directory(dir_name, dir_path, file) + if file == 'benchmark.rb' + # Pattern 2: Single benchmark.rb in directory + BenchmarkEntry.new( + name: dir_name, + script_path: File.join(dir_path, file), + directory: dir_name + ) + else + # Pattern 3: Multiple .rb files (derive suffix from filename without .rb extension) + suffix = file.delete_suffix('.rb') + BenchmarkEntry.new( + name: "#{dir_name}-#{suffix}", + script_path: File.join(dir_path, file), + directory: dir_name + ) + end + end +end diff --git a/lib/benchmark_filter.rb b/lib/benchmark_filter.rb index abbd0392..3219d711 100644 --- a/lib/benchmark_filter.rb +++ b/lib/benchmark_filter.rb @@ -2,16 +2,16 @@ # Filters benchmarks based on categories and name patterns class BenchmarkFilter - def initialize(categories:, name_filters:, excludes:, metadata:) + def initialize(categories:, name_filters:, excludes:, metadata:, directory_map: {}) @categories = categories @name_filters = process_name_filters(name_filters) @excludes = excludes @metadata = metadata @category_cache = {} + @directory_map = directory_map end - def match?(entry) - name = entry.sub(/\.rb\z/, '') + def match?(name) matches_category?(name) && matches_name_filter?(name) && !matches_excludes?(name) end @@ -27,7 +27,27 @@ def matches_category?(name) def matches_name_filter?(name) return true if @name_filters.empty? - @name_filters.any? { |filter| filter === name } + @name_filters.any? do |filter| + if filter.is_a?(Regexp) + filter === name + else + # Exact match + next true if filter == name + + matches_prefix_in_same_directory?(name, filter) + end + end + end + + # Prefix match only for benchmarks in the same directory + # e.g., "addressable" matches "addressable-equality" if they're in the same dir + # but "erubi" does NOT match "erubi-rails" if they're in different dirs + def matches_prefix_in_same_directory?(name, filter) + return false unless name.start_with?("#{filter}-") + + benchmark_dir = @directory_map[name] + # Only match if the benchmark is in a directory with the filter name + benchmark_dir == filter end def matches_excludes?(name) diff --git a/lib/benchmark_suite.rb b/lib/benchmark_suite.rb index 45c5159d..4143e109 100644 --- a/lib/benchmark_suite.rb +++ b/lib/benchmark_suite.rb @@ -9,6 +9,7 @@ require 'rbconfig' require_relative 'benchmark_filter' require_relative 'benchmark_runner' +require_relative 'benchmark_discovery' # BenchmarkSuite runs a collection of benchmarks and collects their results class BenchmarkSuite @@ -41,20 +42,18 @@ def run bench_data = {} bench_failures = {} - bench_file_grouping.each do |bench_dir, bench_files| - bench_files.each_with_index do |entry, idx| - bench_name = entry.delete_suffix('.rb') + benchmark_entries = discover_benchmarks - puts("Running benchmark \"#{bench_name}\" (#{idx+1}/#{bench_files.length})") + benchmark_entries.each_with_index do |entry, idx| + puts("Running benchmark \"#{entry.name}\" (#{idx+1}/#{benchmark_entries.length})") - result_json_path = File.join(out_path, "temp#{Process.pid}.json") - result = run_single_benchmark(bench_dir, entry, result_json_path) + result_json_path = File.join(out_path, "temp#{Process.pid}.json") + result = run_single_benchmark(entry.script_path, result_json_path) - if result[:success] - bench_data[bench_name] = process_benchmark_result(result_json_path, result[:command]) - else - bench_failures[bench_name] = result[:status].exitstatus - end + if result[:success] + bench_data[entry.name] = process_benchmark_result(result_json_path, result[:command]) + else + bench_failures[entry.name] = result[:status].exitstatus end end @@ -82,14 +81,68 @@ def process_benchmark_result(result_json_path, command) end end - def run_single_benchmark(bench_dir, entry, result_json_path) - # Path to the benchmark runner script - script_path = File.join(bench_dir, entry) + def discover_benchmarks + all_entries = discover_all_benchmark_entries + directory_map = build_directory_map(all_entries) + filter_benchmarks(all_entries, directory_map) + end + + def discover_all_benchmark_entries + main_discovery = BenchmarkDiscovery.new(bench_dir) + main_entries = main_discovery.discover + + ractor_entries = if benchmark_ractor_directory? + ractor_discovery = BenchmarkDiscovery.new(ractor_bench_dir) + ractor_discovery.discover + else + [] + end + + { main: main_entries, ractor: ractor_entries } + end - unless script_path.end_with?('.rb') - script_path = File.join(script_path, 'benchmark.rb') + def build_directory_map(all_entries) + combined_entries = all_entries[:main] + all_entries[:ractor] + combined_entries.each_with_object({}) do |entry, map| + map[entry.name] = entry.directory end + end + + def filter_benchmarks(all_entries, directory_map) + main_benchmarks = filter_entries( + all_entries[:main], + categories: categories, + name_filters: name_filters, + excludes: excludes, + directory_map: directory_map + ) + + if benchmark_ractor_directory? + ractor_benchmarks = filter_entries( + all_entries[:ractor], + categories: [], + name_filters: name_filters, + excludes: excludes, + directory_map: directory_map + ) + main_benchmarks + ractor_benchmarks + else + main_benchmarks + end + end + + def filter_entries(entries, categories:, name_filters:, excludes:, directory_map:) + filter = BenchmarkFilter.new( + categories: categories, + name_filters: name_filters, + excludes: excludes, + metadata: benchmarks_metadata, + directory_map: directory_map + ) + entries.select { |entry| filter.match?(entry.name) } + end + def run_single_benchmark(script_path, result_json_path) # Fix for jruby/jruby#7394 in JRuby 9.4.2.0 script_path = File.expand_path(script_path) @@ -132,47 +185,10 @@ def benchmark_env end end - def bench_file_grouping - grouping = { bench_dir => filtered_bench_entries(bench_dir, main_benchmark_filter) } - - if benchmark_ractor_directory? - # We ignore the category filter here because everything in the - # benchmarks-ractor directory should be included when we're benchmarking the - # Ractor category - grouping[ractor_bench_dir] = filtered_bench_entries(ractor_bench_dir, ractor_benchmark_filter) - end - - grouping - end - - def main_benchmark_filter - @main_benchmark_filter ||= BenchmarkFilter.new( - categories: categories, - name_filters: name_filters, - excludes: excludes, - metadata: benchmarks_metadata - ) - end - - def ractor_benchmark_filter - @ractor_benchmark_filter ||= BenchmarkFilter.new( - categories: [], - name_filters: name_filters, - excludes: excludes, - metadata: benchmarks_metadata - ) - end - def benchmarks_metadata @benchmarks_metadata ||= YAML.load_file('benchmarks.yml') end - def filtered_bench_entries(dir, filter) - Dir.children(dir).sort.filter do |entry| - filter.match?(entry) - end - end - def benchmark_ractor_directory? categories == RACTOR_CATEGORY end diff --git a/test/benchmark_discovery_test.rb b/test/benchmark_discovery_test.rb new file mode 100644 index 00000000..516ef155 --- /dev/null +++ b/test/benchmark_discovery_test.rb @@ -0,0 +1,441 @@ +require_relative 'test_helper' +require_relative '../lib/benchmark_discovery' +require 'tmpdir' +require 'fileutils' + +describe BenchmarkDiscovery do + before do + @original_dir = Dir.pwd + @temp_dir = Dir.mktmpdir + Dir.chdir(@temp_dir) + end + + after do + Dir.chdir(@original_dir) + FileUtils.rm_rf(@temp_dir) + end + + describe '#discover' do + it 'returns empty array when directory does not exist' do + discovery = BenchmarkDiscovery.new('nonexistent') + entries = discovery.discover + + assert_equal [], entries + end + + it 'returns empty array when directory is empty' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(bench_dir) + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + assert_equal [], entries + end + + describe 'Pattern 1: Standalone .rb files' do + it 'discovers standalone .rb files in directory' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(bench_dir) + + # Create standalone .rb files + File.write(File.join(bench_dir, 'fib.rb'), '# fib benchmark') + File.write(File.join(bench_dir, 'matmul.rb'), '# matmul benchmark') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + assert_equal 2, entries.length + + assert_equal 'fib', entries[0].name + assert_equal File.join(bench_dir, 'fib.rb'), entries[0].script_path + + assert_equal 'matmul', entries[1].name + assert_equal File.join(bench_dir, 'matmul.rb'), entries[1].script_path + end + + it 'ignores non-.rb files' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(bench_dir) + + File.write(File.join(bench_dir, 'fib.rb'), '# fib benchmark') + File.write(File.join(bench_dir, 'README.md'), '# README') + File.write(File.join(bench_dir, 'data.txt'), 'data') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + assert_equal 1, entries.length + assert_equal 'fib', entries[0].name + end + + it 'handles files with hyphens and underscores in names' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(bench_dir) + + File.write(File.join(bench_dir, '30k_ifelse.rb'), '# benchmark') + File.write(File.join(bench_dir, 'ruby-xor.rb'), '# benchmark') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + assert_equal 2, entries.length + assert_equal '30k_ifelse', entries[0].name + assert_equal 'ruby-xor', entries[1].name + end + end + + describe 'Pattern 2: Single benchmark.rb in directory' do + it 'discovers benchmark.rb in subdirectory' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(File.join(bench_dir, 'erubi')) + + File.write(File.join(bench_dir, 'erubi', 'benchmark.rb'), '# erubi benchmark') + File.write(File.join(bench_dir, 'erubi', 'Gemfile'), 'source "https://rubygems.org"') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + assert_equal 1, entries.length + assert_equal 'erubi', entries[0].name + assert_equal File.join(bench_dir, 'erubi', 'benchmark.rb'), entries[0].script_path + end + + it 'uses directory name as benchmark name' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(File.join(bench_dir, 'my-complex-benchmark')) + + File.write(File.join(bench_dir, 'my-complex-benchmark', 'benchmark.rb'), '# benchmark') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + assert_equal 1, entries.length + assert_equal 'my-complex-benchmark', entries[0].name + end + + it 'discovers multiple directories with benchmark.rb' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(File.join(bench_dir, 'erubi')) + FileUtils.mkdir_p(File.join(bench_dir, 'liquid-render')) + + File.write(File.join(bench_dir, 'erubi', 'benchmark.rb'), '# erubi') + File.write(File.join(bench_dir, 'liquid-render', 'benchmark.rb'), '# liquid') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + assert_equal 2, entries.length + assert_equal 'erubi', entries[0].name + assert_equal 'liquid-render', entries[1].name + end + + it 'ignores directories without benchmark.rb' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(File.join(bench_dir, 'empty-dir')) + FileUtils.mkdir_p(File.join(bench_dir, 'data-dir')) + + File.write(File.join(bench_dir, 'data-dir', 'data.json'), '{}') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + assert_equal 0, entries.length + end + end + + describe 'Pattern 3: Multiple .rb files in directory' do + it 'discovers multiple .rb files in directory' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(File.join(bench_dir, 'addressable')) + + File.write(File.join(bench_dir, 'addressable', 'equality.rb'), '# equality') + File.write(File.join(bench_dir, 'addressable', 'join.rb'), '# join') + File.write(File.join(bench_dir, 'addressable', 'parse.rb'), '# parse') + File.write(File.join(bench_dir, 'addressable', 'Gemfile'), 'source') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + assert_equal 3, entries.length + + # Results should be sorted alphabetically + assert_equal 'addressable-equality', entries[0].name + assert_equal File.join(bench_dir, 'addressable', 'equality.rb'), entries[0].script_path + + assert_equal 'addressable-join', entries[1].name + assert_equal File.join(bench_dir, 'addressable', 'join.rb'), entries[1].script_path + + assert_equal 'addressable-parse', entries[2].name + assert_equal File.join(bench_dir, 'addressable', 'parse.rb'), entries[2].script_path + end + + it 'handles .rb files with complex names' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(File.join(bench_dir, 'test')) + + File.write(File.join(bench_dir, 'test', 'multi-word-suffix.rb'), '# test') + File.write(File.join(bench_dir, 'test', 'with_underscore.rb'), '# test') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + assert_equal 2, entries.length + assert_equal 'test-multi-word-suffix', entries[0].name + assert_equal 'test-with_underscore', entries[1].name + end + + it 'discovers all .rb files except benchmark.rb' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(File.join(bench_dir, 'addressable')) + + File.write(File.join(bench_dir, 'addressable', 'equality.rb'), '# benchmark') + File.write(File.join(bench_dir, 'addressable', 'helper.rb'), '# helper') + File.write(File.join(bench_dir, 'addressable', 'lib.rb'), '# lib') + File.write(File.join(bench_dir, 'addressable', 'Gemfile'), 'source') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + assert_equal 3, entries.length + assert_equal 'addressable-equality', entries[0].name + assert_equal 'addressable-helper', entries[1].name + assert_equal 'addressable-lib', entries[2].name + end + end + + describe 'Mixed patterns' do + it 'discovers all three patterns together' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(bench_dir) + FileUtils.mkdir_p(File.join(bench_dir, 'erubi')) + FileUtils.mkdir_p(File.join(bench_dir, 'addressable')) + + # Pattern 1: Standalone files + File.write(File.join(bench_dir, 'fib.rb'), '# fib') + File.write(File.join(bench_dir, 'matmul.rb'), '# matmul') + + # Pattern 2: Single benchmark.rb + File.write(File.join(bench_dir, 'erubi', 'benchmark.rb'), '# erubi') + + # Pattern 3: Multiple .rb files + File.write(File.join(bench_dir, 'addressable', 'equality.rb'), '# eq') + File.write(File.join(bench_dir, 'addressable', 'join.rb'), '# join') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + assert_equal 5, entries.length + + # Should be sorted: addressable-* comes first, then erubi, then fib, then matmul + names = entries.map(&:name) + assert_equal ['addressable-equality', 'addressable-join', 'erubi', 'fib', 'matmul'], names + end + + it 'handles directories with both benchmark.rb and other .rb files' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(File.join(bench_dir, 'mixed')) + + File.write(File.join(bench_dir, 'mixed', 'benchmark.rb'), '# default') + File.write(File.join(bench_dir, 'mixed', 'variant.rb'), '# variant') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + assert_equal 1, entries.length + assert_equal 'mixed', entries[0].name + end + end + + describe 'Sorting' do + it 'returns entries sorted alphabetically' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(bench_dir) + + # Create files in non-alphabetical order + File.write(File.join(bench_dir, 'zebra.rb'), '# z') + File.write(File.join(bench_dir, 'apple.rb'), '# a') + File.write(File.join(bench_dir, 'monkey.rb'), '# m') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + names = entries.map(&:name) + assert_equal ['apple', 'monkey', 'zebra'], names + end + + it 'sorts directories and files together alphabetically' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(bench_dir) + FileUtils.mkdir_p(File.join(bench_dir, 'bdir')) + FileUtils.mkdir_p(File.join(bench_dir, 'ddir')) + + File.write(File.join(bench_dir, 'afile.rb'), '# a') + File.write(File.join(bench_dir, 'cfile.rb'), '# c') + File.write(File.join(bench_dir, 'bdir', 'benchmark.rb'), '# b') + File.write(File.join(bench_dir, 'ddir', 'benchmark.rb'), '# d') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + names = entries.map(&:name) + assert_equal ['afile', 'bdir', 'cfile', 'ddir'], names + end + + it 'sorts .rb files within a directory alphabetically' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(File.join(bench_dir, 'test')) + + # Create in non-alphabetical order + File.write(File.join(bench_dir, 'test', 'zebra.rb'), '# z') + File.write(File.join(bench_dir, 'test', 'apple.rb'), '# a') + File.write(File.join(bench_dir, 'test', 'monkey.rb'), '# m') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + names = entries.map(&:name) + assert_equal ['test-apple', 'test-monkey', 'test-zebra'], names + end + end + + describe 'Edge cases' do + it 'handles empty subdirectories' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(File.join(bench_dir, 'empty')) + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + assert_equal 0, entries.length + end + + it 'handles deeply nested benchmark files' do + bench_dir = File.join(@temp_dir, 'benchmarks') + # Note: The discovery only looks one level deep + FileUtils.mkdir_p(File.join(bench_dir, 'outer', 'inner')) + + File.write(File.join(bench_dir, 'outer', 'benchmark.rb'), '# outer') + File.write(File.join(bench_dir, 'outer', 'inner', 'benchmark.rb'), '# inner') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + # Should only find the outer benchmark + assert_equal 1, entries.length + assert_equal 'outer', entries[0].name + end + + it 'handles special characters in directory names' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(File.join(bench_dir, 'my-special_bench.mark')) + + File.write(File.join(bench_dir, 'my-special_bench.mark', 'benchmark.rb'), '# test') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + assert_equal 1, entries.length + assert_equal 'my-special_bench.mark', entries[0].name + end + + it 'returns BenchmarkEntry objects with correct attributes' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(bench_dir) + + File.write(File.join(bench_dir, 'test.rb'), '# test') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + assert_equal 1, entries.length + + entry = entries[0] + assert_instance_of BenchmarkDiscovery::BenchmarkEntry, entry + assert_respond_to entry, :name + assert_respond_to entry, :script_path + assert_equal 'test', entry.name + assert_equal File.join(bench_dir, 'test.rb'), entry.script_path + end + end + end + + describe 'Real-world examples' do + it 'matches the addressable directory structure' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(File.join(bench_dir, 'addressable')) + + # Simulate actual addressable benchmarks + [ + 'equality.rb', + 'getters.rb', + 'join.rb', + 'merge.rb', + 'new.rb', + 'normalize.rb', + 'parse.rb', + 'setters.rb', + 'to-s.rb' + ].each do |file| + File.write(File.join(bench_dir, 'addressable', file), '# benchmark') + end + + File.write(File.join(bench_dir, 'addressable', 'Gemfile'), 'source') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + assert_equal 9, entries.length + + expected_names = [ + 'addressable-equality', + 'addressable-getters', + 'addressable-join', + 'addressable-merge', + 'addressable-new', + 'addressable-normalize', + 'addressable-parse', + 'addressable-setters', + 'addressable-to-s' + ] + + assert_equal expected_names, entries.map(&:name) + end + + it 'matches a typical benchmarks directory structure' do + bench_dir = File.join(@temp_dir, 'benchmarks') + FileUtils.mkdir_p(bench_dir) + + # Pattern 1: Standalone files + File.write(File.join(bench_dir, 'fib.rb'), '# fib') + File.write(File.join(bench_dir, '30k_ifelse.rb'), '# ifelse') + + # Pattern 2: Single benchmark.rb + FileUtils.mkdir_p(File.join(bench_dir, 'erubi')) + File.write(File.join(bench_dir, 'erubi', 'benchmark.rb'), '# erubi') + + FileUtils.mkdir_p(File.join(bench_dir, 'liquid-render')) + File.write(File.join(bench_dir, 'liquid-render', 'benchmark.rb'), '# liquid') + + # Pattern 3: Multiple benchmarks + FileUtils.mkdir_p(File.join(bench_dir, 'addressable')) + File.write(File.join(bench_dir, 'addressable', 'parse.rb'), '# parse') + File.write(File.join(bench_dir, 'addressable', 'join.rb'), '# join') + + discovery = BenchmarkDiscovery.new(bench_dir) + entries = discovery.discover + + assert_equal 6, entries.length + + names = entries.map(&:name) + assert_equal [ + '30k_ifelse', + 'addressable-join', + 'addressable-parse', + 'erubi', + 'fib', + 'liquid-render' + ], names + end + end +end diff --git a/test/benchmark_filter_test.rb b/test/benchmark_filter_test.rb index d24490cc..5b623d10 100644 --- a/test/benchmark_filter_test.rb +++ b/test/benchmark_filter_test.rb @@ -1,6 +1,7 @@ require_relative 'test_helper' require_relative '../lib/benchmark_filter' require_relative '../lib/benchmark_runner' +require_relative '../lib/benchmark_discovery' describe BenchmarkFilter do before do @@ -16,82 +17,199 @@ it 'matches when no filters provided' do filter = BenchmarkFilter.new(categories: [], name_filters: [], excludes: [], metadata: @metadata) - assert_equal true, filter.match?('fib.rb') + assert_equal true, filter.match?('fib') end it 'matches by category' do filter = BenchmarkFilter.new(categories: ['micro'], name_filters: [], excludes: [], metadata: @metadata) - assert_equal true, filter.match?('fib.rb') - assert_equal false, filter.match?('railsbench.rb') + assert_equal true, filter.match?('fib') + assert_equal false, filter.match?('railsbench') end it 'matches by name filter' do filter = BenchmarkFilter.new(categories: [], name_filters: ['fib'], excludes: [], metadata: @metadata) - assert_equal true, filter.match?('fib.rb') - assert_equal false, filter.match?('railsbench.rb') + assert_equal true, filter.match?('fib') + assert_equal false, filter.match?('railsbench') end it 'applies excludes' do filter = BenchmarkFilter.new(categories: ['headline'], name_filters: [], excludes: ['railsbench'], metadata: @metadata) - assert_equal true, filter.match?('optcarrot.rb') - assert_equal false, filter.match?('railsbench.rb') + assert_equal true, filter.match?('optcarrot') + assert_equal false, filter.match?('railsbench') end it 'matches ractor category' do filter = BenchmarkFilter.new(categories: ['ractor'], name_filters: [], excludes: [], metadata: @metadata) - assert_equal true, filter.match?('ractor_bench.rb') - end - - it 'strips .rb extension from entry name' do - filter = BenchmarkFilter.new(categories: [], name_filters: ['fib'], excludes: [], metadata: @metadata) - - assert_equal true, filter.match?('fib.rb') + assert_equal true, filter.match?('ractor_bench') end it 'handles regex filters' do filter = BenchmarkFilter.new(categories: [], name_filters: ['/rails/'], excludes: [], metadata: @metadata) - assert_equal true, filter.match?('railsbench.rb') - assert_equal false, filter.match?('fib.rb') + assert_equal true, filter.match?('railsbench') + assert_equal false, filter.match?('fib') end it 'handles case-insensitive regex filters' do filter = BenchmarkFilter.new(categories: [], name_filters: ['/RAILS/i'], excludes: [], metadata: @metadata) - assert_equal true, filter.match?('railsbench.rb') + assert_equal true, filter.match?('railsbench') end it 'handles multiple categories' do filter = BenchmarkFilter.new(categories: ['micro', 'headline'], name_filters: [], excludes: [], metadata: @metadata) - assert_equal true, filter.match?('fib.rb') - assert_equal true, filter.match?('railsbench.rb') + assert_equal true, filter.match?('fib') + assert_equal true, filter.match?('railsbench') end it 'requires both category and name filter to match when both provided' do filter = BenchmarkFilter.new(categories: ['micro'], name_filters: ['rails'], excludes: [], metadata: @metadata) - assert_equal false, filter.match?('fib.rb') # matches category but not name - assert_equal false, filter.match?('railsbench.rb') # matches name but not category + assert_equal false, filter.match?('fib') # matches category but not name + assert_equal false, filter.match?('railsbench') # matches name but not category end it 'handles complex regex patterns' do filter = BenchmarkFilter.new(categories: [], name_filters: ['/opt.*rot/'], excludes: [], metadata: @metadata) - assert_equal true, filter.match?('optcarrot.rb') - assert_equal false, filter.match?('fib.rb') + assert_equal true, filter.match?('optcarrot') + assert_equal false, filter.match?('fib') end it 'handles mixed string and regex filters' do filter = BenchmarkFilter.new(categories: [], name_filters: ['fib', '/rails/'], excludes: [], metadata: @metadata) - assert_equal true, filter.match?('fib.rb') - assert_equal true, filter.match?('railsbench.rb') - assert_equal false, filter.match?('optcarrot.rb') + assert_equal true, filter.match?('fib') + assert_equal true, filter.match?('railsbench') + assert_equal false, filter.match?('optcarrot') + end + + it 'matches directory prefix for multi-benchmark directories' do + metadata = { + 'addressable-equality' => { 'category' => 'other' }, + 'addressable-join' => { 'category' => 'other' }, + 'addressable-parse' => { 'category' => 'other' }, + 'fib' => { 'category' => 'micro' } + } + + filter = BenchmarkFilter.new( + categories: [], + name_filters: ['addressable'], + excludes: [], + metadata: metadata, + directory_map: { + 'addressable-equality' => 'addressable', + 'addressable-join' => 'addressable', + 'addressable-parse' => 'addressable', + 'fib' => nil + } + ) + + assert_equal true, filter.match?('addressable-equality') + assert_equal true, filter.match?('addressable-join') + assert_equal true, filter.match?('addressable-parse') + + assert_equal false, filter.match?('fib') + end + + it 'matches exact name when there is no hyphen' do + metadata = { + 'erubi' => { 'category' => 'other' }, + 'erubi-rails' => { 'category' => 'headline' }, + 'fib' => { 'category' => 'micro' } + } + + filter = BenchmarkFilter.new( + categories: [], + name_filters: ['erubi'], + excludes: [], + metadata: metadata, + directory_map: { + 'erubi' => 'erubi', + 'erubi-rails' => 'erubi-rails', + 'fib' => nil + } + ) + + assert_equal true, filter.match?('erubi') + + assert_equal false, filter.match?('erubi-rails') + + assert_equal false, filter.match?('fib') + end + + it 'allows specific benchmark names with prefix matching disabled via exact match' do + metadata = { + 'addressable-equality' => { 'category' => 'other' }, + 'addressable-join' => { 'category' => 'other' } + } + + filter = BenchmarkFilter.new(categories: [], name_filters: ['addressable-equality'], excludes: [], metadata: metadata) + + assert_equal true, filter.match?('addressable-equality') + + assert_equal false, filter.match?('addressable-join') + end + + it 'prefix matching works with multiple filters' do + metadata = { + 'addressable-equality' => { 'category' => 'other' }, + 'addressable-join' => { 'category' => 'other' }, + 'erubi' => { 'category' => 'other' }, + 'erubi-rails' => { 'category' => 'headline' }, + 'fib' => { 'category' => 'micro' } + } + + filter = BenchmarkFilter.new( + categories: [], + name_filters: ['addressable', 'fib'], + excludes: [], + metadata: metadata, + directory_map: { + 'addressable-equality' => 'addressable', + 'addressable-join' => 'addressable', + 'erubi' => 'erubi', + 'erubi-rails' => 'erubi-rails', + 'fib' => nil + } + ) + + assert_equal true, filter.match?('addressable-equality') + assert_equal true, filter.match?('addressable-join') + + assert_equal true, filter.match?('fib') + + assert_equal false, filter.match?('erubi') + assert_equal false, filter.match?('erubi-rails') + end + + it 'regex filters are not affected by prefix matching logic' do + metadata = { + 'addressable-equality' => { 'category' => 'other' }, + 'addressable-join' => { 'category' => 'other' }, + 'fib' => { 'category' => 'micro' } + } + + filter = BenchmarkFilter.new( + categories: [], + name_filters: ['/addr.*able/'], + excludes: [], + metadata: metadata, + directory_map: { + 'addressable-equality' => 'addressable', + 'addressable-join' => 'addressable', + 'fib' => nil + } + ) + + assert_equal true, filter.match?('addressable-equality') + assert_equal true, filter.match?('addressable-join') + assert_equal false, filter.match?('fib') end end end diff --git a/test/benchmarks_test.rb b/test/benchmarks_test.rb index 654225fa..ea1bb3b1 100644 --- a/test/benchmarks_test.rb +++ b/test/benchmarks_test.rb @@ -1,4 +1,5 @@ require_relative 'test_helper' +require_relative '../lib/benchmark_discovery' require 'yaml' describe 'benchmarks.yml' do @@ -6,10 +7,19 @@ yjit_bench = File.expand_path('..', __dir__) benchmarks_yml = YAML.load_file("#{yjit_bench}/benchmarks.yml") - benchmarks = Dir.glob("#{yjit_bench}/benchmarks*/*").map do |entry| - File.basename(entry).delete_suffix('.rb') - end.compact + # Use BenchmarkDiscovery to find all benchmarks + benchmarks_dir = File.join(yjit_bench, 'benchmarks') + benchmarks_ractor_dir = File.join(yjit_bench, 'benchmarks-ractor') - assert_equal benchmarks.sort, benchmarks_yml.keys.map{ |k| k.sub('ractor/', '') }.sort + benchmarks = [] + benchmarks.concat(BenchmarkDiscovery.new(benchmarks_dir).discover.map(&:name)) + benchmarks.concat(BenchmarkDiscovery.new(benchmarks_ractor_dir).discover.map { |e| "ractor/#{e.name}" }) + + # Compare discovered benchmarks with those in benchmarks.yml + # Note: benchmarks.yml may have entries with "ractor/" prefix which we normalize + yml_keys = benchmarks_yml.keys.sort + discovered_keys = benchmarks.sort + + assert_equal yml_keys, discovered_keys end end