Add deterministic protocol replay harness
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
const std = @import("std");
|
||||
const protocol = @import("protocol.zig");
|
||||
const replay = @import("replay.zig");
|
||||
const session = @import("session.zig");
|
||||
const socket = @import("socket.zig");
|
||||
|
||||
@@ -115,6 +116,7 @@ fn collectRemainingArgs(allocator: std.mem.Allocator, args: *std.process.Args.It
|
||||
|
||||
test {
|
||||
_ = protocol;
|
||||
_ = replay;
|
||||
_ = session;
|
||||
_ = socket;
|
||||
}
|
||||
|
||||
+267
@@ -0,0 +1,267 @@
|
||||
const std = @import("std");
|
||||
const protocol = @import("protocol.zig");
|
||||
const session_mod = @import("session.zig");
|
||||
|
||||
// Deterministic headless replay harness.
|
||||
// req: session/004, testing/001, testing/002, testing/003, testing/004
|
||||
|
||||
test {
|
||||
_ = Recorder;
|
||||
_ = replayText;
|
||||
}
|
||||
|
||||
pub const ReplayError = error{
|
||||
BadReplayLine,
|
||||
Diverged,
|
||||
};
|
||||
|
||||
pub const Divergence = struct {
|
||||
step: usize,
|
||||
message: []const u8,
|
||||
};
|
||||
|
||||
pub const ReplayResult = union(enum) {
|
||||
ok,
|
||||
diverged: Divergence,
|
||||
|
||||
pub fn deinit(self: ReplayResult, allocator: std.mem.Allocator) void {
|
||||
switch (self) {
|
||||
.ok => {},
|
||||
.diverged => |diverged| allocator.free(diverged.message),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
pub const Recorder = struct {
|
||||
allocator: std.mem.Allocator,
|
||||
lines: std.ArrayList(u8) = .empty,
|
||||
|
||||
pub fn init(allocator: std.mem.Allocator) Recorder {
|
||||
return .{ .allocator = allocator };
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Recorder) void {
|
||||
self.lines.deinit(self.allocator);
|
||||
self.* = undefined;
|
||||
}
|
||||
|
||||
pub fn protocolLine(self: *Recorder, line: []const u8) !void {
|
||||
try self.appendLine("protocol", line);
|
||||
}
|
||||
|
||||
pub fn inputInsert(self: *Recorder, input_text: []const u8) !void {
|
||||
try self.appendLine("input insert", input_text);
|
||||
}
|
||||
|
||||
pub fn expectResponse(self: *Recorder, response: []const u8) !void {
|
||||
try self.appendLine("expect-response", response);
|
||||
}
|
||||
|
||||
pub fn saveCheckpoint(self: *Recorder, bytes: []const u8) !void {
|
||||
try self.appendLine("save", bytes);
|
||||
}
|
||||
|
||||
pub fn text(self: *const Recorder) []const u8 {
|
||||
return self.lines.items;
|
||||
}
|
||||
|
||||
fn appendLine(self: *Recorder, tag: []const u8, payload: []const u8) !void {
|
||||
if (std.mem.indexOfScalar(u8, payload, '\n') != null) return ReplayError.BadReplayLine;
|
||||
try self.lines.appendSlice(self.allocator, tag);
|
||||
try self.lines.append(self.allocator, ' ');
|
||||
try self.lines.appendSlice(self.allocator, payload);
|
||||
try self.lines.append(self.allocator, '\n');
|
||||
}
|
||||
};
|
||||
|
||||
pub fn replayText(allocator: std.mem.Allocator, text: []const u8) !ReplayResult {
|
||||
var session = session_mod.Session.init(allocator);
|
||||
defer session.deinit();
|
||||
|
||||
var last_response = std.ArrayList(u8).empty;
|
||||
defer last_response.deinit(allocator);
|
||||
|
||||
var step: usize = 0;
|
||||
var lines = std.mem.splitScalar(u8, text, '\n');
|
||||
while (lines.next()) |raw_line| {
|
||||
if (raw_line.len == 0) continue;
|
||||
step += 1;
|
||||
const result = try replayStep(allocator, &session, &last_response, step, raw_line);
|
||||
switch (result) {
|
||||
.ok => {},
|
||||
.diverged => return result,
|
||||
}
|
||||
}
|
||||
|
||||
return .ok;
|
||||
}
|
||||
|
||||
fn replayStep(
|
||||
allocator: std.mem.Allocator,
|
||||
session: *session_mod.Session,
|
||||
last_response: *std.ArrayList(u8),
|
||||
step: usize,
|
||||
line: []const u8,
|
||||
) !ReplayResult {
|
||||
if (std.mem.startsWith(u8, line, "protocol ")) {
|
||||
const response = try protocol.handleLine(allocator, session, line[9..]);
|
||||
defer allocator.free(response);
|
||||
last_response.clearRetainingCapacity();
|
||||
try last_response.appendSlice(allocator, response);
|
||||
return .ok;
|
||||
}
|
||||
|
||||
if (std.mem.startsWith(u8, line, "input insert ")) {
|
||||
if (!std.unicode.utf8ValidateSlice(line[13..])) {
|
||||
return makeDivergence(allocator, step, "input insert payload is not valid UTF-8");
|
||||
}
|
||||
const command = try std.fmt.allocPrint(allocator, "command insert {s}", .{line[13..]});
|
||||
defer allocator.free(command);
|
||||
const response = try protocol.handleLine(allocator, session, command);
|
||||
defer allocator.free(response);
|
||||
last_response.clearRetainingCapacity();
|
||||
try last_response.appendSlice(allocator, response);
|
||||
return .ok;
|
||||
}
|
||||
|
||||
if (std.mem.startsWith(u8, line, "expect-response ")) {
|
||||
const expected = line[16..];
|
||||
const actual = std.mem.trimEnd(u8, last_response.items, "\n");
|
||||
if (!std.mem.eql(u8, expected, actual)) {
|
||||
return divergenceFmt(
|
||||
allocator,
|
||||
step,
|
||||
"expected response '{s}' but got '{s}'",
|
||||
.{ expected, actual },
|
||||
);
|
||||
}
|
||||
return .ok;
|
||||
}
|
||||
|
||||
if (std.mem.startsWith(u8, line, "save ")) {
|
||||
const expected = line[5..];
|
||||
const snap = session.snapshot() catch |err| switch (err) {
|
||||
error.NoBufferOpen => return makeDivergence(allocator, step, "save checkpoint has no open buffer"),
|
||||
};
|
||||
if (!std.mem.eql(u8, expected, snap.bytes)) {
|
||||
return divergenceFmt(
|
||||
allocator,
|
||||
step,
|
||||
"expected saved bytes '{s}' but got '{s}'",
|
||||
.{ expected, snap.bytes },
|
||||
);
|
||||
}
|
||||
return .ok;
|
||||
}
|
||||
|
||||
return makeDivergence(allocator, step, "unknown replay event");
|
||||
}
|
||||
|
||||
fn makeDivergence(allocator: std.mem.Allocator, step: usize, message: []const u8) !ReplayResult {
|
||||
return .{ .diverged = .{
|
||||
.step = step,
|
||||
.message = try allocator.dupe(u8, message),
|
||||
} };
|
||||
}
|
||||
|
||||
fn divergenceFmt(allocator: std.mem.Allocator, step: usize, comptime fmt: []const u8, args: anytype) !ReplayResult {
|
||||
return .{ .diverged = .{
|
||||
.step = step,
|
||||
.message = try std.fmt.allocPrint(allocator, fmt, args),
|
||||
} };
|
||||
}
|
||||
|
||||
fn expectReplayOk(result: ReplayResult) !void {
|
||||
switch (result) {
|
||||
.ok => {},
|
||||
.diverged => return error.ExpectedReplayOk,
|
||||
}
|
||||
}
|
||||
|
||||
test "regular: golden open move edit save recording replays deterministically" {
|
||||
var recorder = Recorder.init(std.testing.allocator);
|
||||
defer recorder.deinit();
|
||||
|
||||
try recorder.protocolLine("open let café = 1");
|
||||
try recorder.expectResponse("ok state cursor_byte=0 cursor_cell=0 bytes_len=13");
|
||||
try recorder.protocolLine("command move_right");
|
||||
try recorder.protocolLine("command move_right");
|
||||
try recorder.protocolLine("command move_right");
|
||||
try recorder.inputInsert("🔥");
|
||||
try recorder.expectResponse("ok state cursor_byte=7 cursor_cell=5 bytes_len=17");
|
||||
try recorder.saveCheckpoint("let🔥 café = 1");
|
||||
|
||||
const result = try replayText(std.testing.allocator, recorder.text());
|
||||
defer result.deinit(std.testing.allocator);
|
||||
try expectReplayOk(result);
|
||||
}
|
||||
|
||||
test "regular: replay can be rerun with identical result" {
|
||||
const recording =
|
||||
\\protocol open abc
|
||||
\\protocol command move_right
|
||||
\\input insert é
|
||||
\\expect-response ok state cursor_byte=3 cursor_cell=2 bytes_len=5
|
||||
\\save aébc
|
||||
\\
|
||||
;
|
||||
|
||||
const first = try replayText(std.testing.allocator, recording);
|
||||
defer first.deinit(std.testing.allocator);
|
||||
const second = try replayText(std.testing.allocator, recording);
|
||||
defer second.deinit(std.testing.allocator);
|
||||
|
||||
try expectReplayOk(first);
|
||||
try expectReplayOk(second);
|
||||
}
|
||||
|
||||
test "adversarial: replay names first divergent response step" {
|
||||
const recording =
|
||||
\\protocol open abc
|
||||
\\expect-response ok state cursor_byte=9 cursor_cell=9 bytes_len=99
|
||||
\\protocol command move_right
|
||||
\\
|
||||
;
|
||||
|
||||
const result = try replayText(std.testing.allocator, recording);
|
||||
defer result.deinit(std.testing.allocator);
|
||||
switch (result) {
|
||||
.diverged => |diverged| {
|
||||
try std.testing.expectEqual(@as(usize, 2), diverged.step);
|
||||
try std.testing.expect(std.mem.indexOf(u8, diverged.message, "expected response") != null);
|
||||
},
|
||||
.ok => return error.ExpectedReplayDivergence,
|
||||
}
|
||||
}
|
||||
|
||||
test "adversarial: replay names first divergent save step" {
|
||||
const recording =
|
||||
\\protocol open abc
|
||||
\\protocol command move_right
|
||||
\\input insert é
|
||||
\\save wrong
|
||||
\\
|
||||
;
|
||||
|
||||
const result = try replayText(std.testing.allocator, recording);
|
||||
defer result.deinit(std.testing.allocator);
|
||||
switch (result) {
|
||||
.diverged => |diverged| {
|
||||
try std.testing.expectEqual(@as(usize, 4), diverged.step);
|
||||
try std.testing.expect(std.mem.indexOf(u8, diverged.message, "expected saved bytes") != null);
|
||||
},
|
||||
.ok => return error.ExpectedReplayDivergence,
|
||||
}
|
||||
}
|
||||
|
||||
test "adversarial: replay rejects unknown events without mutating future steps" {
|
||||
const result = try replayText(std.testing.allocator, "plugin install everything\n");
|
||||
defer result.deinit(std.testing.allocator);
|
||||
switch (result) {
|
||||
.diverged => |diverged| {
|
||||
try std.testing.expectEqual(@as(usize, 1), diverged.step);
|
||||
try std.testing.expectEqualStrings("unknown replay event", diverged.message);
|
||||
},
|
||||
.ok => return error.ExpectedReplayDivergence,
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user