feat: reporters and CLI suite
- reporter.zig: text table writer (Go-compatible columns, mean ± stddev when --count > 1) and ndjson writer, both driving std.Io.Writer. - suite.zig: Suite holds registered (name, fn) entries, owns name storage, drives run/run_cli, and routes sub-benchmark calls back through a sub_run trampoline using current_name to compose parent/child paths. Parses --filter / --min-time / --count / --max-iters / --allocs / --format / --list / --help, with a small duration parser (1s, 500ms, 100us, 250ns).
This commit is contained in:
123
src/reporter.zig
Normal file
123
src/reporter.zig
Normal file
@@ -0,0 +1,123 @@
|
||||
const std = @import("std");
|
||||
const runner = @import("runner.zig");
|
||||
const stats = @import("stats.zig");
|
||||
|
||||
const Result = runner.Result;
|
||||
const Writer = std.Io.Writer;
|
||||
|
||||
pub const Format = enum { text, json };
|
||||
|
||||
pub const ReportOpts = struct {
|
||||
/// Always print B/op and allocs/op even when zero.
|
||||
always_allocs: bool = false,
|
||||
};
|
||||
|
||||
pub fn write_text_header(w: *Writer) !void {
|
||||
try w.print("{s:<30} {s:>10} {s:>12} {s:>10} {s:>10} {s:>12}\n", .{
|
||||
"benchmark", "iters", "ns/op", "B/op", "allocs/op", "MB/s",
|
||||
});
|
||||
}
|
||||
|
||||
/// Print one benchmark result group (>= 1 attempt) as a text row.
|
||||
pub fn write_text(
|
||||
w: *Writer,
|
||||
name: []const u8,
|
||||
runs: []const Result,
|
||||
opts: ReportOpts,
|
||||
) !void {
|
||||
if (runs.len == 0) return;
|
||||
if (runs[0].is_container) return;
|
||||
|
||||
var buf: [256]f64 = undefined;
|
||||
const ns_samples: []f64 = buf[0..@min(runs.len, buf.len)];
|
||||
for (ns_samples, runs[0..ns_samples.len]) |*s, r| s.* = r.ns_per_op;
|
||||
const ns_summary = stats.summarize(ns_samples);
|
||||
|
||||
const last = runs[runs.len - 1];
|
||||
const show_allocs = opts.always_allocs or last.force_report_allocs or
|
||||
last.bytes_per_op > 0 or last.allocs_per_op > 0;
|
||||
|
||||
try w.print("{s:<30} {d:>10} ", .{ name, last.n });
|
||||
|
||||
if (runs.len > 1) {
|
||||
try w.print("{d:>7.2} \xc2\xb1 {d:<2.2} ", .{ ns_summary.mean, ns_summary.stddev });
|
||||
} else {
|
||||
try w.print("{d:>12.2} ", .{ ns_summary.mean });
|
||||
}
|
||||
|
||||
if (show_allocs) {
|
||||
try w.print("{d:>10.0} {d:>10.0} ", .{ last.bytes_per_op, last.allocs_per_op });
|
||||
} else {
|
||||
try w.print("{s:>10} {s:>10} ", .{ "", "" });
|
||||
}
|
||||
|
||||
if (last.mb_per_sec) |mb| {
|
||||
try w.print("{d:>12.2}", .{mb});
|
||||
} else {
|
||||
try w.print("{s:>12}", .{""});
|
||||
}
|
||||
try w.writeAll("\n");
|
||||
}
|
||||
|
||||
/// Emit one benchmark group as a single ndjson line.
|
||||
pub fn write_json(
|
||||
w: *Writer,
|
||||
name: []const u8,
|
||||
runs: []const Result,
|
||||
) !void {
|
||||
if (runs.len == 0) return;
|
||||
if (runs[0].is_container) return;
|
||||
|
||||
var buf: [256]f64 = undefined;
|
||||
const ns_samples: []f64 = buf[0..@min(runs.len, buf.len)];
|
||||
for (ns_samples, runs[0..ns_samples.len]) |*s, r| s.* = r.ns_per_op;
|
||||
const s = stats.summarize(ns_samples);
|
||||
|
||||
const last = runs[runs.len - 1];
|
||||
|
||||
try w.writeAll("{");
|
||||
try write_json_string_field(w, "name", name);
|
||||
try w.print(",\"n\":{d}", .{last.n});
|
||||
try w.print(",\"ns_per_op\":{d}", .{s.mean});
|
||||
try w.print(",\"bytes_per_op\":{d}", .{last.bytes_per_op});
|
||||
try w.print(",\"allocs_per_op\":{d}", .{last.allocs_per_op});
|
||||
|
||||
if (last.mb_per_sec) |mb| {
|
||||
try w.print(",\"mb_per_sec\":{d}", .{mb});
|
||||
} else {
|
||||
try w.writeAll(",\"mb_per_sec\":null");
|
||||
}
|
||||
|
||||
try w.print(",\"count\":{d}", .{runs.len});
|
||||
|
||||
if (runs.len > 1) {
|
||||
try w.print(
|
||||
",\"ns_per_op_mean\":{d},\"ns_per_op_stddev\":{d},\"ns_per_op_min\":{d}",
|
||||
.{ s.mean, s.stddev, s.min },
|
||||
);
|
||||
try w.writeAll(",\"samples\":[");
|
||||
for (ns_samples, 0..) |x, i| {
|
||||
if (i > 0) try w.writeAll(",");
|
||||
try w.print("{d}", .{x});
|
||||
}
|
||||
try w.writeAll("]");
|
||||
}
|
||||
|
||||
try w.writeAll("}\n");
|
||||
}
|
||||
|
||||
fn write_json_string_field(w: *Writer, key: []const u8, value: []const u8) !void {
|
||||
try w.print("\"{s}\":\"", .{key});
|
||||
for (value) |c| {
|
||||
switch (c) {
|
||||
'"' => try w.writeAll("\\\""),
|
||||
'\\' => try w.writeAll("\\\\"),
|
||||
'\n' => try w.writeAll("\\n"),
|
||||
'\r' => try w.writeAll("\\r"),
|
||||
'\t' => try w.writeAll("\\t"),
|
||||
0x00...0x08, 0x0b, 0x0c, 0x0e...0x1f => try w.print("\\u{x:0>4}", .{c}),
|
||||
else => try w.print("{c}", .{c}),
|
||||
}
|
||||
}
|
||||
try w.writeAll("\"");
|
||||
}
|
||||
262
src/suite.zig
Normal file
262
src/suite.zig
Normal file
@@ -0,0 +1,262 @@
|
||||
const std = @import("std");
|
||||
const Allocator = std.mem.Allocator;
|
||||
const Io = std.Io;
|
||||
const Writer = std.Io.Writer;
|
||||
|
||||
const bench = @import("benchmark.zig");
|
||||
const runner = @import("runner.zig");
|
||||
const reporter = @import("reporter.zig");
|
||||
const CountingAllocator = @import("alloc.zig").CountingAllocator;
|
||||
|
||||
const Benchmark = bench.Benchmark;
|
||||
const BenchFn = bench.BenchFn;
|
||||
const Result = runner.Result;
|
||||
|
||||
pub const Format = reporter.Format;
|
||||
|
||||
pub const Options = struct {
|
||||
min_time_ns: u64 = std.time.ns_per_s,
|
||||
max_iters: u64 = 1_000_000_000,
|
||||
count: u32 = 1,
|
||||
filter: ?[]const u8 = null,
|
||||
format: Format = .text,
|
||||
always_allocs: bool = false,
|
||||
};
|
||||
|
||||
pub const Suite = struct {
|
||||
pub const Entry = struct {
|
||||
name: []const u8, // owned by Suite (gpa-dup'd)
|
||||
f: BenchFn,
|
||||
};
|
||||
|
||||
gpa: Allocator,
|
||||
io: Io,
|
||||
entries: std.ArrayListUnmanaged(Entry) = .empty,
|
||||
|
||||
/// Mutable state during a run — installed by `run`.
|
||||
active_writer: ?*Writer = null,
|
||||
active_opts: Options = .{},
|
||||
active_counter: ?*CountingAllocator = null,
|
||||
/// The composed name of the benchmark currently being executed.
|
||||
/// Used by `sub_run_trampoline` to prefix sub-benchmark names.
|
||||
current_name: ?[]const u8 = null,
|
||||
|
||||
pub fn init(gpa: Allocator, io: Io) Suite {
|
||||
return .{ .gpa = gpa, .io = io };
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Suite) void {
|
||||
for (self.entries.items) |e| self.gpa.free(e.name);
|
||||
self.entries.deinit(self.gpa);
|
||||
self.* = undefined;
|
||||
}
|
||||
|
||||
pub fn add(self: *Suite, name: []const u8, f: BenchFn) !void {
|
||||
const owned = try self.gpa.dupe(u8, name);
|
||||
errdefer self.gpa.free(owned);
|
||||
try self.entries.append(self.gpa, .{ .name = owned, .f = f });
|
||||
}
|
||||
|
||||
/// Run all registered benchmarks with `opts`, writing to `writer`.
|
||||
pub fn run(self: *Suite, opts: Options, writer: *Writer) !void {
|
||||
var counter = CountingAllocator.init(self.gpa);
|
||||
|
||||
self.active_writer = writer;
|
||||
self.active_opts = opts;
|
||||
self.active_counter = &counter;
|
||||
defer {
|
||||
self.active_writer = null;
|
||||
self.active_counter = null;
|
||||
self.current_name = null;
|
||||
}
|
||||
|
||||
if (opts.format == .text) try reporter.write_text_header(writer);
|
||||
|
||||
for (self.entries.items) |entry| {
|
||||
if (opts.filter) |needle| {
|
||||
// Substring match on the top-level name, or on the parent
|
||||
// segment when the user passes a `parent/leaf` style filter.
|
||||
const target = if (std.mem.indexOf(u8, needle, "/")) |i|
|
||||
needle[0..i]
|
||||
else
|
||||
needle;
|
||||
if (std.mem.indexOf(u8, entry.name, target) == null) continue;
|
||||
}
|
||||
try self.run_named(entry.name, entry.f);
|
||||
}
|
||||
|
||||
try writer.flush();
|
||||
}
|
||||
|
||||
/// Run a benchmark by composed name. Used both for top-level entries
|
||||
/// and sub-benchmarks (`parent/sub`).
|
||||
fn run_named(self: *Suite, full_name: []const u8, f: BenchFn) !void {
|
||||
const opts = self.active_opts;
|
||||
const writer = self.active_writer.?;
|
||||
const counter = self.active_counter.?;
|
||||
|
||||
const saved_current = self.current_name;
|
||||
self.current_name = full_name;
|
||||
defer self.current_name = saved_current;
|
||||
|
||||
const count = @max(@as(u32, 1), opts.count);
|
||||
const samples = try self.gpa.alloc(Result, count);
|
||||
defer self.gpa.free(samples);
|
||||
|
||||
var i: u32 = 0;
|
||||
while (i < count) : (i += 1) {
|
||||
samples[i] = try runner.run_one(
|
||||
full_name,
|
||||
f,
|
||||
counter,
|
||||
Suite.sub_run_trampoline,
|
||||
self,
|
||||
self.io,
|
||||
.{ .min_time_ns = opts.min_time_ns, .max_iters = opts.max_iters },
|
||||
);
|
||||
// If the user delegated to sub-benchmarks, no point repeating.
|
||||
if (samples[i].is_container) {
|
||||
i += 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
const used = samples[0..i];
|
||||
|
||||
switch (opts.format) {
|
||||
.text => try reporter.write_text(
|
||||
writer,
|
||||
full_name,
|
||||
used,
|
||||
.{ .always_allocs = opts.always_allocs },
|
||||
),
|
||||
.json => try reporter.write_json(writer, full_name, used),
|
||||
}
|
||||
}
|
||||
|
||||
fn sub_run_trampoline(ctx: *anyopaque, sub_name: []const u8, f: BenchFn) anyerror!void {
|
||||
const self: *Suite = @ptrCast(@alignCast(ctx));
|
||||
const parent = self.current_name orelse "<orphan>";
|
||||
const composed = try std.fmt.allocPrint(self.gpa, "{s}/{s}", .{ parent, sub_name });
|
||||
defer self.gpa.free(composed);
|
||||
|
||||
// Apply filter against the composed name so users can target a
|
||||
// specific sub-benchmark with --filter=parent/leaf or by substring.
|
||||
if (self.active_opts.filter) |needle| {
|
||||
if (std.mem.indexOf(u8, composed, needle) == null) return;
|
||||
}
|
||||
|
||||
try self.run_named(composed, f);
|
||||
}
|
||||
|
||||
/// CLI entrypoint: parse `proc_init.minimal.args`, then dispatch to `run`.
|
||||
pub fn run_cli(self: *Suite, proc_init: std.process.Init) !void {
|
||||
var args_it = std.process.Args.Iterator.init(proc_init.minimal.args);
|
||||
_ = args_it.skip(); // argv[0]
|
||||
|
||||
var opts: Options = .{};
|
||||
var list_only = false;
|
||||
|
||||
while (args_it.next()) |raw| {
|
||||
const arg: []const u8 = raw;
|
||||
if (std.mem.eql(u8, arg, "--help") or std.mem.eql(u8, arg, "-h")) {
|
||||
try print_help(proc_init);
|
||||
return;
|
||||
} else if (std.mem.eql(u8, arg, "--list")) {
|
||||
list_only = true;
|
||||
} else if (std.mem.eql(u8, arg, "--allocs")) {
|
||||
opts.always_allocs = true;
|
||||
} else if (std.mem.startsWith(u8, arg, "--filter=")) {
|
||||
opts.filter = arg["--filter=".len..];
|
||||
} else if (std.mem.startsWith(u8, arg, "--min-time=")) {
|
||||
opts.min_time_ns = try parse_duration_ns(arg["--min-time=".len..]);
|
||||
} else if (std.mem.startsWith(u8, arg, "--count=")) {
|
||||
opts.count = try std.fmt.parseInt(u32, arg["--count=".len..], 10);
|
||||
} else if (std.mem.startsWith(u8, arg, "--max-iters=")) {
|
||||
opts.max_iters = try std.fmt.parseInt(u64, arg["--max-iters=".len..], 10);
|
||||
} else if (std.mem.startsWith(u8, arg, "--format=")) {
|
||||
const v = arg["--format=".len..];
|
||||
if (std.mem.eql(u8, v, "text")) {
|
||||
opts.format = .text;
|
||||
} else if (std.mem.eql(u8, v, "json")) {
|
||||
opts.format = .json;
|
||||
} else return error.InvalidFormat;
|
||||
} else {
|
||||
return error.UnknownFlag;
|
||||
}
|
||||
}
|
||||
|
||||
var buf: [4096]u8 = undefined;
|
||||
var stderr_file = std.Io.File.stderr().writerStreaming(proc_init.io, &buf);
|
||||
|
||||
if (list_only) {
|
||||
for (self.entries.items) |e| {
|
||||
try stderr_file.interface.print("{s}\n", .{e.name});
|
||||
}
|
||||
try stderr_file.interface.flush();
|
||||
return;
|
||||
}
|
||||
|
||||
try self.run(opts, &stderr_file.interface);
|
||||
}
|
||||
};
|
||||
|
||||
fn print_help(proc_init: std.process.Init) !void {
|
||||
var buf: [2048]u8 = undefined;
|
||||
var stderr_file = std.Io.File.stderr().writerStreaming(proc_init.io, &buf);
|
||||
const w = &stderr_file.interface;
|
||||
try w.writeAll(
|
||||
\\zbench — benchmark runner
|
||||
\\
|
||||
\\Flags:
|
||||
\\ --filter=<substring> Only run benchmarks whose name contains <substring>
|
||||
\\ --min-time=<dur> Minimum wall time per benchmark (e.g. 1s, 500ms, 100us)
|
||||
\\ --count=<n> Repeat each benchmark <n> times for statistics
|
||||
\\ --max-iters=<n> Cap on iterations per benchmark
|
||||
\\ --allocs Always print B/op and allocs/op columns
|
||||
\\ --format=text|json Output format (default: text)
|
||||
\\ --list Print names of all benchmarks and exit
|
||||
\\ --help, -h Show this help
|
||||
\\
|
||||
);
|
||||
try w.flush();
|
||||
}
|
||||
|
||||
/// Parses "100ns", "10us", "5ms", "2s" → nanoseconds. Bare numbers are
|
||||
/// interpreted as seconds, matching the "1s" default.
|
||||
pub fn parse_duration_ns(s: []const u8) !u64 {
|
||||
if (s.len == 0) return error.InvalidDuration;
|
||||
|
||||
var split: usize = 0;
|
||||
while (split < s.len) : (split += 1) {
|
||||
const c = s[split];
|
||||
if (!std.ascii.isDigit(c) and c != '.') break;
|
||||
}
|
||||
if (split == 0) return error.InvalidDuration;
|
||||
|
||||
const num_str = s[0..split];
|
||||
const unit = s[split..];
|
||||
|
||||
const value: u64 = try std.fmt.parseInt(u64, num_str, 10);
|
||||
|
||||
if (unit.len == 0 or std.mem.eql(u8, unit, "s")) {
|
||||
return value *| std.time.ns_per_s;
|
||||
} else if (std.mem.eql(u8, unit, "ms")) {
|
||||
return value *| std.time.ns_per_ms;
|
||||
} else if (std.mem.eql(u8, unit, "us") or std.mem.eql(u8, unit, "µs")) {
|
||||
return value *| std.time.ns_per_us;
|
||||
} else if (std.mem.eql(u8, unit, "ns")) {
|
||||
return value;
|
||||
} else {
|
||||
return error.InvalidDuration;
|
||||
}
|
||||
}
|
||||
|
||||
test "parse_duration_ns" {
|
||||
try std.testing.expectEqual(@as(u64, std.time.ns_per_s), try parse_duration_ns("1s"));
|
||||
try std.testing.expectEqual(@as(u64, 500 * std.time.ns_per_ms), try parse_duration_ns("500ms"));
|
||||
try std.testing.expectEqual(@as(u64, 100 * std.time.ns_per_us), try parse_duration_ns("100us"));
|
||||
try std.testing.expectEqual(@as(u64, 250), try parse_duration_ns("250ns"));
|
||||
try std.testing.expectEqual(@as(u64, 2 * std.time.ns_per_s), try parse_duration_ns("2"));
|
||||
try std.testing.expectError(error.InvalidDuration, parse_duration_ns(""));
|
||||
try std.testing.expectError(error.InvalidDuration, parse_duration_ns("10xyz"));
|
||||
}
|
||||
Reference in New Issue
Block a user