feat: reporters and CLI suite
- reporter.zig: text table writer (Go-compatible columns, mean ± stddev when --count > 1) and ndjson writer, both driving std.Io.Writer. - suite.zig: Suite holds registered (name, fn) entries, owns name storage, drives run/run_cli, and routes sub-benchmark calls back through a sub_run trampoline using current_name to compose parent/child paths. Parses --filter / --min-time / --count / --max-iters / --allocs / --format / --list / --help, with a small duration parser (1s, 500ms, 100us, 250ns).
This commit is contained in:
123
src/reporter.zig
Normal file
123
src/reporter.zig
Normal file
@@ -0,0 +1,123 @@
|
|||||||
|
const std = @import("std");
|
||||||
|
const runner = @import("runner.zig");
|
||||||
|
const stats = @import("stats.zig");
|
||||||
|
|
||||||
|
const Result = runner.Result;
|
||||||
|
const Writer = std.Io.Writer;
|
||||||
|
|
||||||
|
pub const Format = enum { text, json };
|
||||||
|
|
||||||
|
pub const ReportOpts = struct {
|
||||||
|
/// Always print B/op and allocs/op even when zero.
|
||||||
|
always_allocs: bool = false,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub fn write_text_header(w: *Writer) !void {
|
||||||
|
try w.print("{s:<30} {s:>10} {s:>12} {s:>10} {s:>10} {s:>12}\n", .{
|
||||||
|
"benchmark", "iters", "ns/op", "B/op", "allocs/op", "MB/s",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Print one benchmark result group (>= 1 attempt) as a text row.
|
||||||
|
pub fn write_text(
|
||||||
|
w: *Writer,
|
||||||
|
name: []const u8,
|
||||||
|
runs: []const Result,
|
||||||
|
opts: ReportOpts,
|
||||||
|
) !void {
|
||||||
|
if (runs.len == 0) return;
|
||||||
|
if (runs[0].is_container) return;
|
||||||
|
|
||||||
|
var buf: [256]f64 = undefined;
|
||||||
|
const ns_samples: []f64 = buf[0..@min(runs.len, buf.len)];
|
||||||
|
for (ns_samples, runs[0..ns_samples.len]) |*s, r| s.* = r.ns_per_op;
|
||||||
|
const ns_summary = stats.summarize(ns_samples);
|
||||||
|
|
||||||
|
const last = runs[runs.len - 1];
|
||||||
|
const show_allocs = opts.always_allocs or last.force_report_allocs or
|
||||||
|
last.bytes_per_op > 0 or last.allocs_per_op > 0;
|
||||||
|
|
||||||
|
try w.print("{s:<30} {d:>10} ", .{ name, last.n });
|
||||||
|
|
||||||
|
if (runs.len > 1) {
|
||||||
|
try w.print("{d:>7.2} \xc2\xb1 {d:<2.2} ", .{ ns_summary.mean, ns_summary.stddev });
|
||||||
|
} else {
|
||||||
|
try w.print("{d:>12.2} ", .{ ns_summary.mean });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (show_allocs) {
|
||||||
|
try w.print("{d:>10.0} {d:>10.0} ", .{ last.bytes_per_op, last.allocs_per_op });
|
||||||
|
} else {
|
||||||
|
try w.print("{s:>10} {s:>10} ", .{ "", "" });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (last.mb_per_sec) |mb| {
|
||||||
|
try w.print("{d:>12.2}", .{mb});
|
||||||
|
} else {
|
||||||
|
try w.print("{s:>12}", .{""});
|
||||||
|
}
|
||||||
|
try w.writeAll("\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Emit one benchmark group as a single ndjson line.
|
||||||
|
pub fn write_json(
|
||||||
|
w: *Writer,
|
||||||
|
name: []const u8,
|
||||||
|
runs: []const Result,
|
||||||
|
) !void {
|
||||||
|
if (runs.len == 0) return;
|
||||||
|
if (runs[0].is_container) return;
|
||||||
|
|
||||||
|
var buf: [256]f64 = undefined;
|
||||||
|
const ns_samples: []f64 = buf[0..@min(runs.len, buf.len)];
|
||||||
|
for (ns_samples, runs[0..ns_samples.len]) |*s, r| s.* = r.ns_per_op;
|
||||||
|
const s = stats.summarize(ns_samples);
|
||||||
|
|
||||||
|
const last = runs[runs.len - 1];
|
||||||
|
|
||||||
|
try w.writeAll("{");
|
||||||
|
try write_json_string_field(w, "name", name);
|
||||||
|
try w.print(",\"n\":{d}", .{last.n});
|
||||||
|
try w.print(",\"ns_per_op\":{d}", .{s.mean});
|
||||||
|
try w.print(",\"bytes_per_op\":{d}", .{last.bytes_per_op});
|
||||||
|
try w.print(",\"allocs_per_op\":{d}", .{last.allocs_per_op});
|
||||||
|
|
||||||
|
if (last.mb_per_sec) |mb| {
|
||||||
|
try w.print(",\"mb_per_sec\":{d}", .{mb});
|
||||||
|
} else {
|
||||||
|
try w.writeAll(",\"mb_per_sec\":null");
|
||||||
|
}
|
||||||
|
|
||||||
|
try w.print(",\"count\":{d}", .{runs.len});
|
||||||
|
|
||||||
|
if (runs.len > 1) {
|
||||||
|
try w.print(
|
||||||
|
",\"ns_per_op_mean\":{d},\"ns_per_op_stddev\":{d},\"ns_per_op_min\":{d}",
|
||||||
|
.{ s.mean, s.stddev, s.min },
|
||||||
|
);
|
||||||
|
try w.writeAll(",\"samples\":[");
|
||||||
|
for (ns_samples, 0..) |x, i| {
|
||||||
|
if (i > 0) try w.writeAll(",");
|
||||||
|
try w.print("{d}", .{x});
|
||||||
|
}
|
||||||
|
try w.writeAll("]");
|
||||||
|
}
|
||||||
|
|
||||||
|
try w.writeAll("}\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_json_string_field(w: *Writer, key: []const u8, value: []const u8) !void {
|
||||||
|
try w.print("\"{s}\":\"", .{key});
|
||||||
|
for (value) |c| {
|
||||||
|
switch (c) {
|
||||||
|
'"' => try w.writeAll("\\\""),
|
||||||
|
'\\' => try w.writeAll("\\\\"),
|
||||||
|
'\n' => try w.writeAll("\\n"),
|
||||||
|
'\r' => try w.writeAll("\\r"),
|
||||||
|
'\t' => try w.writeAll("\\t"),
|
||||||
|
0x00...0x08, 0x0b, 0x0c, 0x0e...0x1f => try w.print("\\u{x:0>4}", .{c}),
|
||||||
|
else => try w.print("{c}", .{c}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try w.writeAll("\"");
|
||||||
|
}
|
||||||
262
src/suite.zig
Normal file
262
src/suite.zig
Normal file
@@ -0,0 +1,262 @@
|
|||||||
|
const std = @import("std");
|
||||||
|
const Allocator = std.mem.Allocator;
|
||||||
|
const Io = std.Io;
|
||||||
|
const Writer = std.Io.Writer;
|
||||||
|
|
||||||
|
const bench = @import("benchmark.zig");
|
||||||
|
const runner = @import("runner.zig");
|
||||||
|
const reporter = @import("reporter.zig");
|
||||||
|
const CountingAllocator = @import("alloc.zig").CountingAllocator;
|
||||||
|
|
||||||
|
const Benchmark = bench.Benchmark;
|
||||||
|
const BenchFn = bench.BenchFn;
|
||||||
|
const Result = runner.Result;
|
||||||
|
|
||||||
|
pub const Format = reporter.Format;
|
||||||
|
|
||||||
|
pub const Options = struct {
|
||||||
|
min_time_ns: u64 = std.time.ns_per_s,
|
||||||
|
max_iters: u64 = 1_000_000_000,
|
||||||
|
count: u32 = 1,
|
||||||
|
filter: ?[]const u8 = null,
|
||||||
|
format: Format = .text,
|
||||||
|
always_allocs: bool = false,
|
||||||
|
};
|
||||||
|
|
||||||
|
pub const Suite = struct {
|
||||||
|
pub const Entry = struct {
|
||||||
|
name: []const u8, // owned by Suite (gpa-dup'd)
|
||||||
|
f: BenchFn,
|
||||||
|
};
|
||||||
|
|
||||||
|
gpa: Allocator,
|
||||||
|
io: Io,
|
||||||
|
entries: std.ArrayListUnmanaged(Entry) = .empty,
|
||||||
|
|
||||||
|
/// Mutable state during a run — installed by `run`.
|
||||||
|
active_writer: ?*Writer = null,
|
||||||
|
active_opts: Options = .{},
|
||||||
|
active_counter: ?*CountingAllocator = null,
|
||||||
|
/// The composed name of the benchmark currently being executed.
|
||||||
|
/// Used by `sub_run_trampoline` to prefix sub-benchmark names.
|
||||||
|
current_name: ?[]const u8 = null,
|
||||||
|
|
||||||
|
pub fn init(gpa: Allocator, io: Io) Suite {
|
||||||
|
return .{ .gpa = gpa, .io = io };
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn deinit(self: *Suite) void {
|
||||||
|
for (self.entries.items) |e| self.gpa.free(e.name);
|
||||||
|
self.entries.deinit(self.gpa);
|
||||||
|
self.* = undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn add(self: *Suite, name: []const u8, f: BenchFn) !void {
|
||||||
|
const owned = try self.gpa.dupe(u8, name);
|
||||||
|
errdefer self.gpa.free(owned);
|
||||||
|
try self.entries.append(self.gpa, .{ .name = owned, .f = f });
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run all registered benchmarks with `opts`, writing to `writer`.
|
||||||
|
pub fn run(self: *Suite, opts: Options, writer: *Writer) !void {
|
||||||
|
var counter = CountingAllocator.init(self.gpa);
|
||||||
|
|
||||||
|
self.active_writer = writer;
|
||||||
|
self.active_opts = opts;
|
||||||
|
self.active_counter = &counter;
|
||||||
|
defer {
|
||||||
|
self.active_writer = null;
|
||||||
|
self.active_counter = null;
|
||||||
|
self.current_name = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (opts.format == .text) try reporter.write_text_header(writer);
|
||||||
|
|
||||||
|
for (self.entries.items) |entry| {
|
||||||
|
if (opts.filter) |needle| {
|
||||||
|
// Substring match on the top-level name, or on the parent
|
||||||
|
// segment when the user passes a `parent/leaf` style filter.
|
||||||
|
const target = if (std.mem.indexOf(u8, needle, "/")) |i|
|
||||||
|
needle[0..i]
|
||||||
|
else
|
||||||
|
needle;
|
||||||
|
if (std.mem.indexOf(u8, entry.name, target) == null) continue;
|
||||||
|
}
|
||||||
|
try self.run_named(entry.name, entry.f);
|
||||||
|
}
|
||||||
|
|
||||||
|
try writer.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Run a benchmark by composed name. Used both for top-level entries
|
||||||
|
/// and sub-benchmarks (`parent/sub`).
|
||||||
|
fn run_named(self: *Suite, full_name: []const u8, f: BenchFn) !void {
|
||||||
|
const opts = self.active_opts;
|
||||||
|
const writer = self.active_writer.?;
|
||||||
|
const counter = self.active_counter.?;
|
||||||
|
|
||||||
|
const saved_current = self.current_name;
|
||||||
|
self.current_name = full_name;
|
||||||
|
defer self.current_name = saved_current;
|
||||||
|
|
||||||
|
const count = @max(@as(u32, 1), opts.count);
|
||||||
|
const samples = try self.gpa.alloc(Result, count);
|
||||||
|
defer self.gpa.free(samples);
|
||||||
|
|
||||||
|
var i: u32 = 0;
|
||||||
|
while (i < count) : (i += 1) {
|
||||||
|
samples[i] = try runner.run_one(
|
||||||
|
full_name,
|
||||||
|
f,
|
||||||
|
counter,
|
||||||
|
Suite.sub_run_trampoline,
|
||||||
|
self,
|
||||||
|
self.io,
|
||||||
|
.{ .min_time_ns = opts.min_time_ns, .max_iters = opts.max_iters },
|
||||||
|
);
|
||||||
|
// If the user delegated to sub-benchmarks, no point repeating.
|
||||||
|
if (samples[i].is_container) {
|
||||||
|
i += 1;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const used = samples[0..i];
|
||||||
|
|
||||||
|
switch (opts.format) {
|
||||||
|
.text => try reporter.write_text(
|
||||||
|
writer,
|
||||||
|
full_name,
|
||||||
|
used,
|
||||||
|
.{ .always_allocs = opts.always_allocs },
|
||||||
|
),
|
||||||
|
.json => try reporter.write_json(writer, full_name, used),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sub_run_trampoline(ctx: *anyopaque, sub_name: []const u8, f: BenchFn) anyerror!void {
|
||||||
|
const self: *Suite = @ptrCast(@alignCast(ctx));
|
||||||
|
const parent = self.current_name orelse "<orphan>";
|
||||||
|
const composed = try std.fmt.allocPrint(self.gpa, "{s}/{s}", .{ parent, sub_name });
|
||||||
|
defer self.gpa.free(composed);
|
||||||
|
|
||||||
|
// Apply filter against the composed name so users can target a
|
||||||
|
// specific sub-benchmark with --filter=parent/leaf or by substring.
|
||||||
|
if (self.active_opts.filter) |needle| {
|
||||||
|
if (std.mem.indexOf(u8, composed, needle) == null) return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try self.run_named(composed, f);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// CLI entrypoint: parse `proc_init.minimal.args`, then dispatch to `run`.
|
||||||
|
pub fn run_cli(self: *Suite, proc_init: std.process.Init) !void {
|
||||||
|
var args_it = std.process.Args.Iterator.init(proc_init.minimal.args);
|
||||||
|
_ = args_it.skip(); // argv[0]
|
||||||
|
|
||||||
|
var opts: Options = .{};
|
||||||
|
var list_only = false;
|
||||||
|
|
||||||
|
while (args_it.next()) |raw| {
|
||||||
|
const arg: []const u8 = raw;
|
||||||
|
if (std.mem.eql(u8, arg, "--help") or std.mem.eql(u8, arg, "-h")) {
|
||||||
|
try print_help(proc_init);
|
||||||
|
return;
|
||||||
|
} else if (std.mem.eql(u8, arg, "--list")) {
|
||||||
|
list_only = true;
|
||||||
|
} else if (std.mem.eql(u8, arg, "--allocs")) {
|
||||||
|
opts.always_allocs = true;
|
||||||
|
} else if (std.mem.startsWith(u8, arg, "--filter=")) {
|
||||||
|
opts.filter = arg["--filter=".len..];
|
||||||
|
} else if (std.mem.startsWith(u8, arg, "--min-time=")) {
|
||||||
|
opts.min_time_ns = try parse_duration_ns(arg["--min-time=".len..]);
|
||||||
|
} else if (std.mem.startsWith(u8, arg, "--count=")) {
|
||||||
|
opts.count = try std.fmt.parseInt(u32, arg["--count=".len..], 10);
|
||||||
|
} else if (std.mem.startsWith(u8, arg, "--max-iters=")) {
|
||||||
|
opts.max_iters = try std.fmt.parseInt(u64, arg["--max-iters=".len..], 10);
|
||||||
|
} else if (std.mem.startsWith(u8, arg, "--format=")) {
|
||||||
|
const v = arg["--format=".len..];
|
||||||
|
if (std.mem.eql(u8, v, "text")) {
|
||||||
|
opts.format = .text;
|
||||||
|
} else if (std.mem.eql(u8, v, "json")) {
|
||||||
|
opts.format = .json;
|
||||||
|
} else return error.InvalidFormat;
|
||||||
|
} else {
|
||||||
|
return error.UnknownFlag;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var buf: [4096]u8 = undefined;
|
||||||
|
var stderr_file = std.Io.File.stderr().writerStreaming(proc_init.io, &buf);
|
||||||
|
|
||||||
|
if (list_only) {
|
||||||
|
for (self.entries.items) |e| {
|
||||||
|
try stderr_file.interface.print("{s}\n", .{e.name});
|
||||||
|
}
|
||||||
|
try stderr_file.interface.flush();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
try self.run(opts, &stderr_file.interface);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
fn print_help(proc_init: std.process.Init) !void {
|
||||||
|
var buf: [2048]u8 = undefined;
|
||||||
|
var stderr_file = std.Io.File.stderr().writerStreaming(proc_init.io, &buf);
|
||||||
|
const w = &stderr_file.interface;
|
||||||
|
try w.writeAll(
|
||||||
|
\\zbench — benchmark runner
|
||||||
|
\\
|
||||||
|
\\Flags:
|
||||||
|
\\ --filter=<substring> Only run benchmarks whose name contains <substring>
|
||||||
|
\\ --min-time=<dur> Minimum wall time per benchmark (e.g. 1s, 500ms, 100us)
|
||||||
|
\\ --count=<n> Repeat each benchmark <n> times for statistics
|
||||||
|
\\ --max-iters=<n> Cap on iterations per benchmark
|
||||||
|
\\ --allocs Always print B/op and allocs/op columns
|
||||||
|
\\ --format=text|json Output format (default: text)
|
||||||
|
\\ --list Print names of all benchmarks and exit
|
||||||
|
\\ --help, -h Show this help
|
||||||
|
\\
|
||||||
|
);
|
||||||
|
try w.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parses "100ns", "10us", "5ms", "2s" → nanoseconds. Bare numbers are
|
||||||
|
/// interpreted as seconds, matching the "1s" default.
|
||||||
|
pub fn parse_duration_ns(s: []const u8) !u64 {
|
||||||
|
if (s.len == 0) return error.InvalidDuration;
|
||||||
|
|
||||||
|
var split: usize = 0;
|
||||||
|
while (split < s.len) : (split += 1) {
|
||||||
|
const c = s[split];
|
||||||
|
if (!std.ascii.isDigit(c) and c != '.') break;
|
||||||
|
}
|
||||||
|
if (split == 0) return error.InvalidDuration;
|
||||||
|
|
||||||
|
const num_str = s[0..split];
|
||||||
|
const unit = s[split..];
|
||||||
|
|
||||||
|
const value: u64 = try std.fmt.parseInt(u64, num_str, 10);
|
||||||
|
|
||||||
|
if (unit.len == 0 or std.mem.eql(u8, unit, "s")) {
|
||||||
|
return value *| std.time.ns_per_s;
|
||||||
|
} else if (std.mem.eql(u8, unit, "ms")) {
|
||||||
|
return value *| std.time.ns_per_ms;
|
||||||
|
} else if (std.mem.eql(u8, unit, "us") or std.mem.eql(u8, unit, "µs")) {
|
||||||
|
return value *| std.time.ns_per_us;
|
||||||
|
} else if (std.mem.eql(u8, unit, "ns")) {
|
||||||
|
return value;
|
||||||
|
} else {
|
||||||
|
return error.InvalidDuration;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
test "parse_duration_ns" {
|
||||||
|
try std.testing.expectEqual(@as(u64, std.time.ns_per_s), try parse_duration_ns("1s"));
|
||||||
|
try std.testing.expectEqual(@as(u64, 500 * std.time.ns_per_ms), try parse_duration_ns("500ms"));
|
||||||
|
try std.testing.expectEqual(@as(u64, 100 * std.time.ns_per_us), try parse_duration_ns("100us"));
|
||||||
|
try std.testing.expectEqual(@as(u64, 250), try parse_duration_ns("250ns"));
|
||||||
|
try std.testing.expectEqual(@as(u64, 2 * std.time.ns_per_s), try parse_duration_ns("2"));
|
||||||
|
try std.testing.expectError(error.InvalidDuration, parse_duration_ns(""));
|
||||||
|
try std.testing.expectError(error.InvalidDuration, parse_duration_ns("10xyz"));
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user