From 108c145bdd8c7f6924e0c41adb634eae90baf9c9 Mon Sep 17 00:00:00 2001 From: Manlio Perillo Date: Thu, 20 Apr 2023 18:45:09 +0200 Subject: test: refactorize the code Currently, if there is an error when creating the patches/healed directory, the error message will be printed on stderr, but the build runner will report the test as being successful. Add the fail function and the FailStep, so that the error will be correctly handled by the build runner. Remove the PatchStep, and instead add the heal function so that all the exercises are healed before starting the tests. The heal function executes at the configuration phase, but the possible error is handled by the build runner. --- test/tests.zig | 80 +++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 31 deletions(-) (limited to 'test/tests.zig') diff --git a/test/tests.zig b/test/tests.zig index c8f4af2..5bd1e82 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -2,8 +2,10 @@ const std = @import("std"); const root = @import("../build.zig"); const debug = std.debug; +const fmt = std.fmt; const fs = std.fs; +const Allocator = std.mem.Allocator; const Build = std.build; const Step = Build.Step; const RunStep = std.Build.RunStep; @@ -18,9 +20,10 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { const outdir = "patches/healed"; fs.cwd().makePath(outdir) catch |err| { - debug.print("unable to make '{s}': {s}\n", .{ outdir, @errorName(err) }); - - return step; + return fail(step, "unable to make '{s}': {s}\n", .{ outdir, @errorName(err) }); + }; + heal(b.allocator, exercises, outdir) catch |err| { + return fail(step, "unable to heal exercises: {s}\n", .{@errorName(err)}); }; { @@ -32,14 +35,11 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { i += 1; if (ex.skip) continue; - const patch = PatchStep.create(b, ex, outdir); - const cmd = b.addSystemCommand( &.{ b.zig_exe, "build", b.fmt("-Dn={}", .{i}), "-Dhealed", "test" }, ); cmd.setName(b.fmt("zig build -D={} -Dhealed test", .{i})); cmd.expectExitCode(0); - cmd.step.dependOn(&patch.step); // Some exercise output has an extra space character. if (ex.check_stdout) @@ -93,28 +93,21 @@ fn createCase(b: *Build, name: []const u8) *Step { return case_step; } -// Apply a patch to the specified exercise. -const PatchStep = struct { - const join = fs.path.join; - - const exercises_path = "exercises"; - const patches_path = "patches/patches"; - +// A step that will fail. +const FailStep = struct { step: Step, - exercise: Exercise, - outdir: []const u8, + error_msg: []const u8, - pub fn create(owner: *Build, exercise: Exercise, outdir: []const u8) *PatchStep { - const self = owner.allocator.create(PatchStep) catch @panic("OOM"); + pub fn create(owner: *Build, error_msg: []const u8) *FailStep { + const self = owner.allocator.create(FailStep) catch @panic("OOM"); self.* = .{ .step = Step.init(.{ .id = .custom, - .name = owner.fmt("patch {s}", .{exercise.main_file}), + .name = "fail", .owner = owner, .makeFn = make, }), - .exercise = exercise, - .outdir = outdir, + .error_msg = error_msg, }; return self; @@ -122,25 +115,50 @@ const PatchStep = struct { fn make(step: *Step, _: *std.Progress.Node) !void { const b = step.owner; - const self = @fieldParentPtr(PatchStep, "step", step); - const exercise = self.exercise; - const name = exercise.baseName(); + const self = @fieldParentPtr(FailStep, "step", step); + + try step.result_error_msgs.append(b.allocator, self.error_msg); + return error.MakeFailed; + } +}; + +// A variant of `std.Build.Step.fail` that does not return an error so that it +// can be used in the configuration phase. It returns a FailStep, so that the +// error will be cleanly handled by the build runner. +fn fail(step: *Step, comptime format: []const u8, args: anytype) *Step { + const b = step.owner; + + const fail_step = FailStep.create(b, b.fmt(format, args)); + step.dependOn(&fail_step.step); + + return step; +} + +// Heals all the exercises. +fn heal(allocator: Allocator, exercises: []const Exercise, outdir: []const u8) !void { + const join = fs.path.join; + + const exercises_path = "exercises"; + const patches_path = "patches/patches"; + + for (exercises) |ex| { + const name = ex.baseName(); // Use the POSIX patch variant. - const file = join(b.allocator, &.{ exercises_path, exercise.main_file }) catch - @panic("OOM"); - const patch = join(b.allocator, &.{ patches_path, b.fmt("{s}.patch", .{name}) }) catch - @panic("OOM"); - const output = join(b.allocator, &.{ self.outdir, exercise.main_file }) catch - @panic("OOM"); + const file = try join(allocator, &.{ exercises_path, ex.main_file }); + const patch = b: { + const patch_name = try fmt.allocPrint(allocator, "{s}.patch", .{name}); + break :b try join(allocator, &.{ patches_path, patch_name }); + }; + const output = try join(allocator, &.{ outdir, ex.main_file }); const argv = &.{ "patch", "-i", patch, "-o", output, file }; - var child = std.process.Child.init(argv, b.allocator); + var child = std.process.Child.init(argv, allocator); child.stdout_behavior = .Ignore; // the POSIX standard says that stdout is not used _ = try child.spawnAndWait(); } -}; +} // // Missing functions from std.Build.RunStep -- cgit v1.2.3 From cec0aa51dbb9bff30f871d3f605bc1f0fbc6b4a4 Mon Sep 17 00:00:00 2001 From: Manlio Perillo Date: Thu, 20 Apr 2023 21:13:13 +0200 Subject: test: fix incorrect cleanup code The current cleanup code is incorrect, since it may delete the healed directory while one test case is running. The solution is to make each test case isolate, with its own setup and teardown. Unfortunately it is currently not possible, since each test case modify the same directory. Disable the cleanup step, until a better solution is found. --- test/tests.zig | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'test/tests.zig') diff --git a/test/tests.zig b/test/tests.zig index 5bd1e82..015b55a 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -7,8 +7,8 @@ const fs = std.fs; const Allocator = std.mem.Allocator; const Build = std.build; -const Step = Build.Step; const RunStep = std.Build.RunStep; +const Step = Build.Step; const Exercise = root.Exercise; @@ -27,9 +27,9 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { }; { + // Test that `zig build -Dn=n -Dhealed test` selects the nth exercise. const case_step = createCase(b, "case-1"); - // Test that `zig build -Dn=n -Dhealed test` selects the nth exercise. var i: usize = 0; for (exercises[0 .. exercises.len - 1]) |ex| { i += 1; @@ -54,9 +54,9 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { } { + // Test that `zig build -Dn=n -Dhealed test` skips disabled esercises. const case_step = createCase(b, "case-2"); - // Test that `zig build -Dn=n -Dhealed test` skips disabled esercises. var i: usize = 0; for (exercises[0 .. exercises.len - 1]) |ex| { i += 1; @@ -76,8 +76,10 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { step.dependOn(case_step); } - const cleanup = b.addRemoveDirTree(outdir); - step.dependOn(&cleanup.step); + // Don't add the cleanup step, since it may delete outdir while a test case + // is running. + //const cleanup = b.addRemoveDirTree(outdir); + //step.dependOn(&cleanup.step); return step; } -- cgit v1.2.3 From 131772edd7f475b3b2331117ed1d833ab926a156 Mon Sep 17 00:00:00 2001 From: Manlio Perillo Date: Sat, 22 Apr 2023 17:22:34 +0200 Subject: test: add test for `zig build` and `zig build -Dn=1 start` Add tests for `zig build` and `zig build -Dn=1 start`, in order to test that the all the exercises are processed in the correct order. --- test/tests.zig | 166 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) (limited to 'test/tests.zig') diff --git a/test/tests.zig b/test/tests.zig index 015b55a..ee38f49 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -4,9 +4,12 @@ const root = @import("../build.zig"); const debug = std.debug; const fmt = std.fmt; const fs = std.fs; +const mem = std.mem; const Allocator = std.mem.Allocator; const Build = std.build; +const FileSource = std.Build.FileSource; +const Reader = fs.File.Reader; const RunStep = std.Build.RunStep; const Step = Build.Step; @@ -76,6 +79,45 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { step.dependOn(case_step); } + { + // Test that `zig build -Dhealed` process all the exercises in order. + const case_step = createCase(b, "case-3"); + + // TODO: when an exercise is modified, the cache is not invalidated. + const cmd = b.addSystemCommand(&.{ b.zig_exe, "build", "-Dhealed" }); + cmd.setName("zig build -Dhealed"); + cmd.expectExitCode(0); + + const stderr = cmd.captureStdErr(); + const verify = CheckStep.create(b, exercises, stderr, true); + verify.step.dependOn(&cmd.step); + + case_step.dependOn(&verify.step); + + step.dependOn(case_step); + } + + { + // Test that `zig build -Dhealed -Dn=1 start` process all the exercises + // in order. + const case_step = createCase(b, "case-4"); + + // TODO: when an exercise is modified, the cache is not invalidated. + const cmd = b.addSystemCommand( + &.{ b.zig_exe, "build", "-Dhealed", "-Dn=1", "start" }, + ); + cmd.setName("zig build -Dhealed -Dn=1 start"); + cmd.expectExitCode(0); + + const stderr = cmd.captureStdErr(); + const verify = CheckStep.create(b, exercises, stderr, false); + verify.step.dependOn(&cmd.step); + + case_step.dependOn(&verify.step); + + step.dependOn(case_step); + } + // Don't add the cleanup step, since it may delete outdir while a test case // is running. //const cleanup = b.addRemoveDirTree(outdir); @@ -95,6 +137,130 @@ fn createCase(b: *Build, name: []const u8) *Step { return case_step; } +// Check the output of `zig build` or `zig build -Dn=1 start`. +const CheckStep = struct { + step: Step, + exercises: []const Exercise, + stderr: FileSource, + has_logo: bool, + + pub fn create( + owner: *Build, + exercises: []const Exercise, + stderr: FileSource, + has_logo: bool, + ) *CheckStep { + const self = owner.allocator.create(CheckStep) catch @panic("OOM"); + self.* = .{ + .step = Step.init(.{ + .id = .custom, + .name = "check", + .owner = owner, + .makeFn = make, + }), + .exercises = exercises, + .stderr = stderr, + .has_logo = has_logo, + }; + + return self; + } + + fn make(step: *Step, _: *std.Progress.Node) !void { + const b = step.owner; + const self = @fieldParentPtr(CheckStep, "step", step); + const exercises = self.exercises; + + const stderr_file = try fs.cwd().openFile( + self.stderr.getPath(b), + .{ .mode = .read_only }, + ); + defer stderr_file.close(); + + const stderr = stderr_file.reader(); + for (exercises) |ex| { + if (ex.number() == 1 and self.has_logo) { + // Skip the logo. + var buf: [80]u8 = undefined; + + var lineno: usize = 0; + while (lineno < 8) : (lineno += 1) { + _ = try readLine(stderr, &buf); + } + } + try check_output(step, ex, stderr); + } + } + + fn check_output(step: *Step, exercise: Exercise, reader: Reader) !void { + const b = step.owner; + + var buf: [1024]u8 = undefined; + if (exercise.skip) { + { + const actual = try readLine(reader, &buf) orelse "EOF"; + const expect = b.fmt("Skipping {s}", .{exercise.main_file}); + try check(step, exercise, expect, actual); + } + + { + const actual = try readLine(reader, &buf) orelse "EOF"; + try check(step, exercise, "", actual); + } + + return; + } + + { + const actual = try readLine(reader, &buf) orelse "EOF"; + const expect = b.fmt("Compiling {s}...", .{exercise.main_file}); + try check(step, exercise, expect, actual); + } + + { + const actual = try readLine(reader, &buf) orelse "EOF"; + const expect = b.fmt("Checking {s}...", .{exercise.main_file}); + try check(step, exercise, expect, actual); + } + + { + const actual = try readLine(reader, &buf) orelse "EOF"; + const expect = "PASSED:"; + try check(step, exercise, expect, actual); + } + + // Skip the exercise output. + const nlines = 1 + mem.count(u8, exercise.output, "\n") + 1; + var lineno: usize = 0; + while (lineno < nlines) : (lineno += 1) { + _ = try readLine(reader, &buf) orelse @panic("EOF"); + } + } + + fn check( + step: *Step, + exercise: Exercise, + expect: []const u8, + actual: []const u8, + ) !void { + if (!mem.eql(u8, expect, actual)) { + return step.fail("{s}: expected to see \"{s}\", found \"{s}\"", .{ + exercise.main_file, + expect, + actual, + }); + } + } + + fn readLine(reader: fs.File.Reader, buf: []u8) !?[]const u8 { + if (try reader.readUntilDelimiterOrEof(buf, '\n')) |line| { + return mem.trimRight(u8, line, " \r\n"); + } + + return null; + } +}; + // A step that will fail. const FailStep = struct { step: Step, -- cgit v1.2.3 From 5b2e8421578a02379799415ace72acb85f573bcd Mon Sep 17 00:00:00 2001 From: Manlio Perillo Date: Mon, 24 Apr 2023 12:19:00 +0200 Subject: test: change the order of `zig build` options In test case 1 and 2, move the -Dhealed option before the -Dn option, for consistency. Fix a typo in cmd.setName in test case 1 and 2. Remove a confusing comment in test case 1. --- test/tests.zig | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'test/tests.zig') diff --git a/test/tests.zig b/test/tests.zig index ee38f49..069c3b6 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -30,7 +30,7 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { }; { - // Test that `zig build -Dn=n -Dhealed test` selects the nth exercise. + // Test that `zig build -Dhealed -Dn=n test` selects the nth exercise. const case_step = createCase(b, "case-1"); var i: usize = 0; @@ -39,12 +39,11 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { if (ex.skip) continue; const cmd = b.addSystemCommand( - &.{ b.zig_exe, "build", b.fmt("-Dn={}", .{i}), "-Dhealed", "test" }, + &.{ b.zig_exe, "build", "-Dhealed", b.fmt("-Dn={}", .{i}), "test" }, ); - cmd.setName(b.fmt("zig build -D={} -Dhealed test", .{i})); + cmd.setName(b.fmt("zig build -Dhealed -Dn={} test", .{i})); cmd.expectExitCode(0); - // Some exercise output has an extra space character. if (ex.check_stdout) expectStdOutMatch(cmd, ex.output) else @@ -57,7 +56,7 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { } { - // Test that `zig build -Dn=n -Dhealed test` skips disabled esercises. + // Test that `zig build -Dhealed -Dn=n test` skips disabled esercises. const case_step = createCase(b, "case-2"); var i: usize = 0; @@ -66,9 +65,9 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { if (!ex.skip) continue; const cmd = b.addSystemCommand( - &.{ b.zig_exe, "build", b.fmt("-Dn={}", .{i}), "-Dhealed", "test" }, + &.{ b.zig_exe, "build", "-Dhealed", b.fmt("-Dn={}", .{i}), "test" }, ); - cmd.setName(b.fmt("zig build -D={} -Dhealed test", .{i})); + cmd.setName(b.fmt("zig build -Dhealed -Dn={} test", .{i})); cmd.expectExitCode(0); cmd.expectStdOutEqual(""); expectStdErrMatch(cmd, b.fmt("{s} skipped", .{ex.main_file})); -- cgit v1.2.3 From 4f690b074a4a2eed4f7062515dfcccae6d1725c8 Mon Sep 17 00:00:00 2001 From: Manlio Perillo Date: Mon, 24 Apr 2023 13:03:52 +0200 Subject: test: add a test for `zig build -Dn=1` Add a test for `zig build -Dn=1` in order to test that a broken exercise will print an hint. --- test/tests.zig | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'test/tests.zig') diff --git a/test/tests.zig b/test/tests.zig index 069c3b6..f91c4fd 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -117,6 +117,20 @@ pub fn addCliTests(b: *std.Build, exercises: []const Exercise) *Step { step.dependOn(case_step); } + { + // Test that `zig build -Dn=1` prints the hint. + const case_step = createCase(b, "case-5"); + + const cmd = b.addSystemCommand(&.{ b.zig_exe, "build", "-Dn=1" }); + cmd.setName("zig build -Dn=1"); + cmd.expectExitCode(1); + expectStdErrMatch(cmd, exercises[0].hint); + + case_step.dependOn(&cmd.step); + + step.dependOn(case_step); + } + // Don't add the cleanup step, since it may delete outdir while a test case // is running. //const cleanup = b.addRemoveDirTree(outdir); -- cgit v1.2.3