Merge pull request #13411 from dweiller/custom-test-runner

Custom test runner

lib/std/build.zig

@@ -1509,6 +1509,7 @@ pub const LibExeObjStep = struct {
     name_prefix: []const u8,
     filter: ?[]const u8,
     test_evented_io: bool = false,
+    test_runner: ?[]const u8,
     code_model: std.builtin.CodeModel = .default,
     wasi_exec_model: ?std.builtin.WasiExecModel = null,
     /// Symbols to be exported when compiling to wasm
@@ -1771,6 +1772,7 @@ pub const LibExeObjStep = struct {
             .exec_cmd_args = null,
             .name_prefix = "",
             .filter = null,
+            .test_runner = null,
             .disable_stack_probing = false,
             .disable_sanitize_c = false,
             .sanitize_thread = false,
@@ -2204,6 +2206,11 @@ pub const LibExeObjStep = struct {
         self.filter = if (text) |t| self.builder.dupe(t) else null;
     }
 
+    pub fn setTestRunner(self: *LibExeObjStep, path: ?[]const u8) void {
+        assert(self.kind == .@"test" or self.kind == .test_exe);
+        self.test_runner = if (path) |p| self.builder.dupePath(p) else null;
+    }
+
     /// Handy when you have many C/C++ source files and want them all to have the same flags.
     pub fn addCSourceFiles(self: *LibExeObjStep, files: []const []const u8, flags: []const []const u8) void {
         const c_source_files = self.builder.allocator.create(CSourceFiles) catch unreachable;
@@ -2669,6 +2676,11 @@ pub const LibExeObjStep = struct {
             try zig_args.append(self.name_prefix);
         }
 
+        if (self.test_runner) |test_runner| {
+            try zig_args.append("--test-runner");
+            try zig_args.append(builder.pathFromRoot(test_runner));
+        }
+
         for (builder.debug_log_scopes) |log_scope| {
             try zig_args.append("--debug-log");
             try zig_args.append(log_scope);

src/Compilation.zig

@@ -994,6 +994,7 @@ pub const InitOptions = struct {
     reference_trace: ?u32 = null,
     test_filter: ?[]const u8 = null,
     test_name_prefix: ?[]const u8 = null,
+    test_runner_path: ?[]const u8 = null,
     subsystem: ?std.Target.SubSystem = null,
     /// WASI-only. Type of WASI execution model ("command" or "reactor").
     wasi_exec_model: ?std.builtin.WasiExecModel = null,
@@ -1581,12 +1582,15 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
             errdefer std_pkg.destroy(gpa);
 
             const root_pkg = if (options.is_test) root_pkg: {
-                const test_pkg = try Package.createWithDir(
-                    gpa,
-                    options.zig_lib_directory,
-                    null,
-                    "test_runner.zig",
-                );
+                const test_pkg = if (options.test_runner_path) |test_runner|
+                    try Package.create(gpa, null, test_runner)
+                else
+                    try Package.createWithDir(
+                        gpa,
+                        options.zig_lib_directory,
+                        null,
+                        "test_runner.zig",
+                    );
                 errdefer test_pkg.destroy(gpa);
 
                 break :root_pkg test_pkg;

src/main.zig

@@ -503,6 +503,7 @@ const usage_build_generic =
     \\  --test-cmd-bin                 Appends test binary path to test cmd args
     \\  --test-evented-io              Runs the test in evented I/O mode
     \\  --test-no-exec                 Compiles test binary without running it
+    \\  --test-runner [path]           Specify a custom test runner
     \\
     \\Debug Options (Zig Compiler Development):
     \\  -ftime-report                Print timing diagnostics
@@ -726,6 +727,7 @@ fn buildOutputType(
     var runtime_args_start: ?usize = null;
     var test_filter: ?[]const u8 = null;
     var test_name_prefix: ?[]const u8 = null;
+    var test_runner_path: ?[]const u8 = null;
     var override_local_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_LOCAL_CACHE_DIR");
     var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR");
     var override_lib_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_LIB_DIR");
@@ -1043,6 +1045,8 @@ fn buildOutputType(
                         test_filter = args_iter.nextOrFatal();
                     } else if (mem.eql(u8, arg, "--test-name-prefix")) {
                         test_name_prefix = args_iter.nextOrFatal();
+                    } else if (mem.eql(u8, arg, "--test-runner")) {
+                        test_runner_path = args_iter.nextOrFatal();
                     } else if (mem.eql(u8, arg, "--test-cmd")) {
                         try test_exec_args.append(args_iter.nextOrFatal());
                     } else if (mem.eql(u8, arg, "--cache-dir")) {
@@ -2943,6 +2947,7 @@ fn buildOutputType(
         .test_evented_io = test_evented_io,
         .test_filter = test_filter,
         .test_name_prefix = test_name_prefix,
+        .test_runner_path = test_runner_path,
         .disable_lld_caching = !have_enable_cache,
         .subsystem = subsystem,
         .wasi_exec_model = wasi_exec_model,

test/standalone.zig

@@ -15,6 +15,7 @@ pub fn addCases(cases: *tests.StandaloneContext) void {
     cases.add("test/standalone/main_return_error/error_u8_non_zero.zig");
     cases.add("test/standalone/noreturn_call/inline.zig");
     cases.add("test/standalone/noreturn_call/as_arg.zig");
+    cases.addBuildFile("test/standalone/test_runner_path/build.zig", .{ .requires_stage2 = true });
     cases.addBuildFile("test/standalone/main_pkg_path/build.zig", .{});
     cases.addBuildFile("test/standalone/shared_library/build.zig", .{});
     cases.addBuildFile("test/standalone/mix_o_files/build.zig", .{});

test/standalone/test_runner_path/build.zig

@@ -0,0 +1,11 @@
+const Builder = @import("std").build.Builder;
+
+pub fn build(b: *Builder) void {
+    const test_exe = b.addTestExe("test", "test.zig");
+    test_exe.test_runner = "test_runner.zig";
+
+    const test_run = test_exe.run();
+
+    const test_step = b.step("test", "Test the program");
+    test_step.dependOn(&test_run.step);
+}

test/standalone/test_runner_path/test.zig

@@ -0,0 +1,9 @@
+test "test runner path pass" {}
+
+test "test runner path fail" {
+    return error.Fail;
+}
+
+test "test runner path skip" {
+    return error.SkipZigTest;
+}

test/standalone/test_runner_path/test_runner.zig

@@ -0,0 +1,52 @@
+const std = @import("std");
+const io = std.io;
+const builtin = @import("builtin");
+
+pub const io_mode: io.Mode = builtin.test_io_mode;
+
+pub fn main() void {
+    const test_fn_list = builtin.test_functions;
+    var ok_count: usize = 0;
+    var skip_count: usize = 0;
+    var fail_count: usize = 0;
+
+    var async_frame_buffer: []align(std.Target.stack_align) u8 = undefined;
+    // TODO this is on the next line (using `undefined` above) because otherwise zig incorrectly
+    // ignores the alignment of the slice.
+    async_frame_buffer = &[_]u8{};
+
+    for (test_fn_list) |test_fn| {
+        const result = if (test_fn.async_frame_size) |size| switch (io_mode) {
+            .evented => blk: {
+                if (async_frame_buffer.len < size) {
+                    std.heap.page_allocator.free(async_frame_buffer);
+                    async_frame_buffer = std.heap.page_allocator.alignedAlloc(u8, std.Target.stack_align, size) catch @panic("out of memory");
+                }
+                const casted_fn = @ptrCast(fn () callconv(.Async) anyerror!void, test_fn.func);
+                break :blk await @asyncCall(async_frame_buffer, {}, casted_fn, .{});
+            },
+            .blocking => {
+                skip_count += 1;
+                continue;
+            },
+        } else test_fn.func();
+        if (result) |_| {
+            ok_count += 1;
+        } else |err| switch (err) {
+            error.SkipZigTest => {
+                skip_count += 1;
+            },
+            else => {
+                fail_count += 1;
+            },
+        }
+    }
+    if (ok_count == test_fn_list.len) {
+        std.debug.print("All {d} tests passed.\n", .{ok_count});
+    } else {
+        std.debug.print("{d} passed; {d} skipped; {d} failed.\n", .{ ok_count, skip_count, fail_count });
+    }
+    if (ok_count != 1 or skip_count != 1 or fail_count != 1) {
+        std.process.exit(1);
+    }
+}