From 4f5f69354721a5c6adb3ab78f5803676dfd0225d Mon Sep 17 00:00:00 2001 From: Aleksey Shakhmatov Date: Thu, 21 May 2026 08:13:08 +0300 Subject: [PATCH] feat: reporters and CLI suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - reporter.zig: text table writer (Go-compatible columns, mean ± stddev when --count > 1) and ndjson writer, both driving std.Io.Writer. - suite.zig: Suite holds registered (name, fn) entries, owns name storage, drives run/run_cli, and routes sub-benchmark calls back through a sub_run trampoline using current_name to compose parent/child paths. Parses --filter / --min-time / --count / --max-iters / --allocs / --format / --list / --help, with a small duration parser (1s, 500ms, 100us, 250ns). --- src/reporter.zig | 123 ++++++++++++++++++++++ src/suite.zig | 262 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 385 insertions(+) create mode 100644 src/reporter.zig create mode 100644 src/suite.zig diff --git a/src/reporter.zig b/src/reporter.zig new file mode 100644 index 0000000..dddd9b1 --- /dev/null +++ b/src/reporter.zig @@ -0,0 +1,123 @@ +const std = @import("std"); +const runner = @import("runner.zig"); +const stats = @import("stats.zig"); + +const Result = runner.Result; +const Writer = std.Io.Writer; + +pub const Format = enum { text, json }; + +pub const ReportOpts = struct { + /// Always print B/op and allocs/op even when zero. + always_allocs: bool = false, +}; + +pub fn write_text_header(w: *Writer) !void { + try w.print("{s:<30} {s:>10} {s:>12} {s:>10} {s:>10} {s:>12}\n", .{ + "benchmark", "iters", "ns/op", "B/op", "allocs/op", "MB/s", + }); +} + +/// Print one benchmark result group (>= 1 attempt) as a text row. +pub fn write_text( + w: *Writer, + name: []const u8, + runs: []const Result, + opts: ReportOpts, +) !void { + if (runs.len == 0) return; + if (runs[0].is_container) return; + + var buf: [256]f64 = undefined; + const ns_samples: []f64 = buf[0..@min(runs.len, buf.len)]; + for (ns_samples, runs[0..ns_samples.len]) |*s, r| s.* = r.ns_per_op; + const ns_summary = stats.summarize(ns_samples); + + const last = runs[runs.len - 1]; + const show_allocs = opts.always_allocs or last.force_report_allocs or + last.bytes_per_op > 0 or last.allocs_per_op > 0; + + try w.print("{s:<30} {d:>10} ", .{ name, last.n }); + + if (runs.len > 1) { + try w.print("{d:>7.2} \xc2\xb1 {d:<2.2} ", .{ ns_summary.mean, ns_summary.stddev }); + } else { + try w.print("{d:>12.2} ", .{ ns_summary.mean }); + } + + if (show_allocs) { + try w.print("{d:>10.0} {d:>10.0} ", .{ last.bytes_per_op, last.allocs_per_op }); + } else { + try w.print("{s:>10} {s:>10} ", .{ "", "" }); + } + + if (last.mb_per_sec) |mb| { + try w.print("{d:>12.2}", .{mb}); + } else { + try w.print("{s:>12}", .{""}); + } + try w.writeAll("\n"); +} + +/// Emit one benchmark group as a single ndjson line. +pub fn write_json( + w: *Writer, + name: []const u8, + runs: []const Result, +) !void { + if (runs.len == 0) return; + if (runs[0].is_container) return; + + var buf: [256]f64 = undefined; + const ns_samples: []f64 = buf[0..@min(runs.len, buf.len)]; + for (ns_samples, runs[0..ns_samples.len]) |*s, r| s.* = r.ns_per_op; + const s = stats.summarize(ns_samples); + + const last = runs[runs.len - 1]; + + try w.writeAll("{"); + try write_json_string_field(w, "name", name); + try w.print(",\"n\":{d}", .{last.n}); + try w.print(",\"ns_per_op\":{d}", .{s.mean}); + try w.print(",\"bytes_per_op\":{d}", .{last.bytes_per_op}); + try w.print(",\"allocs_per_op\":{d}", .{last.allocs_per_op}); + + if (last.mb_per_sec) |mb| { + try w.print(",\"mb_per_sec\":{d}", .{mb}); + } else { + try w.writeAll(",\"mb_per_sec\":null"); + } + + try w.print(",\"count\":{d}", .{runs.len}); + + if (runs.len > 1) { + try w.print( + ",\"ns_per_op_mean\":{d},\"ns_per_op_stddev\":{d},\"ns_per_op_min\":{d}", + .{ s.mean, s.stddev, s.min }, + ); + try w.writeAll(",\"samples\":["); + for (ns_samples, 0..) |x, i| { + if (i > 0) try w.writeAll(","); + try w.print("{d}", .{x}); + } + try w.writeAll("]"); + } + + try w.writeAll("}\n"); +} + +fn write_json_string_field(w: *Writer, key: []const u8, value: []const u8) !void { + try w.print("\"{s}\":\"", .{key}); + for (value) |c| { + switch (c) { + '"' => try w.writeAll("\\\""), + '\\' => try w.writeAll("\\\\"), + '\n' => try w.writeAll("\\n"), + '\r' => try w.writeAll("\\r"), + '\t' => try w.writeAll("\\t"), + 0x00...0x08, 0x0b, 0x0c, 0x0e...0x1f => try w.print("\\u{x:0>4}", .{c}), + else => try w.print("{c}", .{c}), + } + } + try w.writeAll("\""); +} diff --git a/src/suite.zig b/src/suite.zig new file mode 100644 index 0000000..f14406a --- /dev/null +++ b/src/suite.zig @@ -0,0 +1,262 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const Io = std.Io; +const Writer = std.Io.Writer; + +const bench = @import("benchmark.zig"); +const runner = @import("runner.zig"); +const reporter = @import("reporter.zig"); +const CountingAllocator = @import("alloc.zig").CountingAllocator; + +const Benchmark = bench.Benchmark; +const BenchFn = bench.BenchFn; +const Result = runner.Result; + +pub const Format = reporter.Format; + +pub const Options = struct { + min_time_ns: u64 = std.time.ns_per_s, + max_iters: u64 = 1_000_000_000, + count: u32 = 1, + filter: ?[]const u8 = null, + format: Format = .text, + always_allocs: bool = false, +}; + +pub const Suite = struct { + pub const Entry = struct { + name: []const u8, // owned by Suite (gpa-dup'd) + f: BenchFn, + }; + + gpa: Allocator, + io: Io, + entries: std.ArrayListUnmanaged(Entry) = .empty, + + /// Mutable state during a run — installed by `run`. + active_writer: ?*Writer = null, + active_opts: Options = .{}, + active_counter: ?*CountingAllocator = null, + /// The composed name of the benchmark currently being executed. + /// Used by `sub_run_trampoline` to prefix sub-benchmark names. + current_name: ?[]const u8 = null, + + pub fn init(gpa: Allocator, io: Io) Suite { + return .{ .gpa = gpa, .io = io }; + } + + pub fn deinit(self: *Suite) void { + for (self.entries.items) |e| self.gpa.free(e.name); + self.entries.deinit(self.gpa); + self.* = undefined; + } + + pub fn add(self: *Suite, name: []const u8, f: BenchFn) !void { + const owned = try self.gpa.dupe(u8, name); + errdefer self.gpa.free(owned); + try self.entries.append(self.gpa, .{ .name = owned, .f = f }); + } + + /// Run all registered benchmarks with `opts`, writing to `writer`. + pub fn run(self: *Suite, opts: Options, writer: *Writer) !void { + var counter = CountingAllocator.init(self.gpa); + + self.active_writer = writer; + self.active_opts = opts; + self.active_counter = &counter; + defer { + self.active_writer = null; + self.active_counter = null; + self.current_name = null; + } + + if (opts.format == .text) try reporter.write_text_header(writer); + + for (self.entries.items) |entry| { + if (opts.filter) |needle| { + // Substring match on the top-level name, or on the parent + // segment when the user passes a `parent/leaf` style filter. + const target = if (std.mem.indexOf(u8, needle, "/")) |i| + needle[0..i] + else + needle; + if (std.mem.indexOf(u8, entry.name, target) == null) continue; + } + try self.run_named(entry.name, entry.f); + } + + try writer.flush(); + } + + /// Run a benchmark by composed name. Used both for top-level entries + /// and sub-benchmarks (`parent/sub`). + fn run_named(self: *Suite, full_name: []const u8, f: BenchFn) !void { + const opts = self.active_opts; + const writer = self.active_writer.?; + const counter = self.active_counter.?; + + const saved_current = self.current_name; + self.current_name = full_name; + defer self.current_name = saved_current; + + const count = @max(@as(u32, 1), opts.count); + const samples = try self.gpa.alloc(Result, count); + defer self.gpa.free(samples); + + var i: u32 = 0; + while (i < count) : (i += 1) { + samples[i] = try runner.run_one( + full_name, + f, + counter, + Suite.sub_run_trampoline, + self, + self.io, + .{ .min_time_ns = opts.min_time_ns, .max_iters = opts.max_iters }, + ); + // If the user delegated to sub-benchmarks, no point repeating. + if (samples[i].is_container) { + i += 1; + break; + } + } + const used = samples[0..i]; + + switch (opts.format) { + .text => try reporter.write_text( + writer, + full_name, + used, + .{ .always_allocs = opts.always_allocs }, + ), + .json => try reporter.write_json(writer, full_name, used), + } + } + + fn sub_run_trampoline(ctx: *anyopaque, sub_name: []const u8, f: BenchFn) anyerror!void { + const self: *Suite = @ptrCast(@alignCast(ctx)); + const parent = self.current_name orelse ""; + const composed = try std.fmt.allocPrint(self.gpa, "{s}/{s}", .{ parent, sub_name }); + defer self.gpa.free(composed); + + // Apply filter against the composed name so users can target a + // specific sub-benchmark with --filter=parent/leaf or by substring. + if (self.active_opts.filter) |needle| { + if (std.mem.indexOf(u8, composed, needle) == null) return; + } + + try self.run_named(composed, f); + } + + /// CLI entrypoint: parse `proc_init.minimal.args`, then dispatch to `run`. + pub fn run_cli(self: *Suite, proc_init: std.process.Init) !void { + var args_it = std.process.Args.Iterator.init(proc_init.minimal.args); + _ = args_it.skip(); // argv[0] + + var opts: Options = .{}; + var list_only = false; + + while (args_it.next()) |raw| { + const arg: []const u8 = raw; + if (std.mem.eql(u8, arg, "--help") or std.mem.eql(u8, arg, "-h")) { + try print_help(proc_init); + return; + } else if (std.mem.eql(u8, arg, "--list")) { + list_only = true; + } else if (std.mem.eql(u8, arg, "--allocs")) { + opts.always_allocs = true; + } else if (std.mem.startsWith(u8, arg, "--filter=")) { + opts.filter = arg["--filter=".len..]; + } else if (std.mem.startsWith(u8, arg, "--min-time=")) { + opts.min_time_ns = try parse_duration_ns(arg["--min-time=".len..]); + } else if (std.mem.startsWith(u8, arg, "--count=")) { + opts.count = try std.fmt.parseInt(u32, arg["--count=".len..], 10); + } else if (std.mem.startsWith(u8, arg, "--max-iters=")) { + opts.max_iters = try std.fmt.parseInt(u64, arg["--max-iters=".len..], 10); + } else if (std.mem.startsWith(u8, arg, "--format=")) { + const v = arg["--format=".len..]; + if (std.mem.eql(u8, v, "text")) { + opts.format = .text; + } else if (std.mem.eql(u8, v, "json")) { + opts.format = .json; + } else return error.InvalidFormat; + } else { + return error.UnknownFlag; + } + } + + var buf: [4096]u8 = undefined; + var stderr_file = std.Io.File.stderr().writerStreaming(proc_init.io, &buf); + + if (list_only) { + for (self.entries.items) |e| { + try stderr_file.interface.print("{s}\n", .{e.name}); + } + try stderr_file.interface.flush(); + return; + } + + try self.run(opts, &stderr_file.interface); + } +}; + +fn print_help(proc_init: std.process.Init) !void { + var buf: [2048]u8 = undefined; + var stderr_file = std.Io.File.stderr().writerStreaming(proc_init.io, &buf); + const w = &stderr_file.interface; + try w.writeAll( + \\zbench — benchmark runner + \\ + \\Flags: + \\ --filter= Only run benchmarks whose name contains + \\ --min-time= Minimum wall time per benchmark (e.g. 1s, 500ms, 100us) + \\ --count= Repeat each benchmark times for statistics + \\ --max-iters= Cap on iterations per benchmark + \\ --allocs Always print B/op and allocs/op columns + \\ --format=text|json Output format (default: text) + \\ --list Print names of all benchmarks and exit + \\ --help, -h Show this help + \\ + ); + try w.flush(); +} + +/// Parses "100ns", "10us", "5ms", "2s" → nanoseconds. Bare numbers are +/// interpreted as seconds, matching the "1s" default. +pub fn parse_duration_ns(s: []const u8) !u64 { + if (s.len == 0) return error.InvalidDuration; + + var split: usize = 0; + while (split < s.len) : (split += 1) { + const c = s[split]; + if (!std.ascii.isDigit(c) and c != '.') break; + } + if (split == 0) return error.InvalidDuration; + + const num_str = s[0..split]; + const unit = s[split..]; + + const value: u64 = try std.fmt.parseInt(u64, num_str, 10); + + if (unit.len == 0 or std.mem.eql(u8, unit, "s")) { + return value *| std.time.ns_per_s; + } else if (std.mem.eql(u8, unit, "ms")) { + return value *| std.time.ns_per_ms; + } else if (std.mem.eql(u8, unit, "us") or std.mem.eql(u8, unit, "µs")) { + return value *| std.time.ns_per_us; + } else if (std.mem.eql(u8, unit, "ns")) { + return value; + } else { + return error.InvalidDuration; + } +} + +test "parse_duration_ns" { + try std.testing.expectEqual(@as(u64, std.time.ns_per_s), try parse_duration_ns("1s")); + try std.testing.expectEqual(@as(u64, 500 * std.time.ns_per_ms), try parse_duration_ns("500ms")); + try std.testing.expectEqual(@as(u64, 100 * std.time.ns_per_us), try parse_duration_ns("100us")); + try std.testing.expectEqual(@as(u64, 250), try parse_duration_ns("250ns")); + try std.testing.expectEqual(@as(u64, 2 * std.time.ns_per_s), try parse_duration_ns("2")); + try std.testing.expectError(error.InvalidDuration, parse_duration_ns("")); + try std.testing.expectError(error.InvalidDuration, parse_duration_ns("10xyz")); +}