ghostty/src/tripwire.zig

//! A library for injecting failures into Zig code for the express
//! purpose of testing error handling paths.
//!
//! Improper `errdefer` is one of the highest sources of bugs in Zig code.
//! Many `errdefer` points are hard to exercise in unit tests and rare
//! to encounter in production, so they often hide bugs. Worse, error
//! scenarios are most likely to put your code in an unexpected state
//! that can result in future assertion failures or memory safety issues.
//!
//! This module aims to solve this problem by providing a way to inject
//! errors at specific points in your code during unit tests, allowing you
//! to test every possible error path.
//!
//! # Usage
//!
//! To use this package, create a `tripwire.module` for each failable
//! function you want to test. The enum must be hand-curated to be the
//! set of fail points, and the error set comes directly from the function
//! itself.
//!
//! Pepper your function with `try tw.check` calls wherever you want to
//! have a testable failure point. You don't need every "try" to have
//! an associated tripwire check, only the ones you care about testing.
//! Usually, this is going to be the points where you want to test
//! errdefer logic above it.
//!
//! In unit tests, add `try tw.errorAlways` or related calls to
//! configure expected failures. Then, call your function. Finally, always
//! call `try tw.end(.reset)` to verify your expectations were met and
//! to reset the tripwire module for future tests.
//!
//! ```
//! const tw = tripwire.module(enum { alloc_buf, open_file }, myFunction);
//!
//! fn myFunction() tw.Error!void {
//!     try tw.check(.alloc_buf);
//!     const buf = try allocator.alloc(u8, 1024);
//!     errdefer allocator.free(buf);
//!
//!     try tw.check(.open_file);
//!     const file = try std.fs.cwd().openFile("foo.txt", .{});
//!     // ...
//! }
//!
//! test "myFunction fails on alloc" {
//!     tw.errorAlways(.alloc_buf, error.OutOfMemory);
//!     try std.testing.expectError(error.OutOfMemory, myFunction());
//!     try tw.end(.reset);
//! }
//! ```
//!
//! ## Transitive Function Calls
//!
//! To test transitive calls, there are two schools of thought:
//!
//!   1. Put a failure point above the transitive call in the caller
//!      and assume the child function error handling works correctly.
//!
//!   2. Create another tripwire module for the child function and
//!      trigger failures there. This is recommended if the child function
//!      can't really be called in isolation (e.g. its an auxiliary function
//!      to a public API).
//!
//! Either works, its situationally dependent which is better.

const std = @import("std");
const builtin = @import("builtin");
const assert = std.debug.assert;
const testing = std.testing;

const log = std.log.scoped(.tripwire);

// Future ideas:
//
//   - Assert that the errors are actually tripped. e.g. you set a
//     errorAlways on a point, and want to verify it was tripped.
//   - Assert that every point is covered by at least one test. We
//     can probably use global state for this.
//   - Error only after a certain number of calls to a point.
//   - Error only on a range of calls (first 5 times, 2-7th time, etc.)
//
// I don't want to implement these until they're actually needed by
// some part of our codebase, but I want to list them here in case they
// do become useful.

/// A tripwire module that can be used to inject failures at specific points.
///
/// Outside of unit tests, this module is free and completely optimized away.
/// It takes up zero binary or runtime space and all function calls are
/// optimized out.
///
/// To use this module, add `check` (or related) calls prior to every
/// `try` operation that you want to be able to fail arbitrarily. Then,
/// in your unit tests, call the `error` family of functions to configure
/// when errors should be injected.
///
/// P is an enum type representing the failure points in the module.
/// E is the error set of possible errors that can be returned from the
/// failure points. You can use `anyerror` here but note you may have to
/// use `checkConstrained` to narrow down the error type when you call
/// it in your function (so your function can compile).
///
/// E may also be an error union type, in which case the error set of that
/// union is used as the error set for the tripwire module.
/// If E is a function, then the error set of the return value of that
/// function is used as the error set for the tripwire module.
pub fn module(
    comptime P: type,
    comptime E: anytype,
) type {
    return struct {
        /// The points this module can fail at.
        pub const FailPoint = P;

        /// The error set used for failures at the failure points.
        pub const Error = err: {
            const T = if (@TypeOf(E) == type) E else @TypeOf(E);
            break :err switch (@typeInfo(T)) {
                .error_set => E,
                .error_union => |info| info.error_set,
                .@"fn" => |info| @typeInfo(info.return_type.?).error_union.error_set,
                else => @compileError("E must be an error set or function type"),
            };
        };

        /// Whether our module is enabled or not. In the future we may
        /// want to make this a comptime parameter to the module.
        pub const enabled = builtin.is_test;

        comptime {
            assert(@typeInfo(FailPoint) == .@"enum");
            assert(@typeInfo(Error) == .error_set);
        }

        /// The configured tripwires for this module.
        var tripwires: TripwireMap = .{};
        const TripwireMap = std.EnumMap(FailPoint, Tripwire);
        const Tripwire = struct {
            /// Error to return when tripped
            err: Error,

            /// The amount of times this tripwire has been reached. This
            /// is NOT the number of times it has tripped, since we may
            /// have mins for that.
            reached: usize = 0,

            /// The minimum number of times this must be reached before
            /// tripping. After this point, it trips every time. This is
            /// a "before" check so if this is "1" then it'll trip the
            /// second time it's reached.
            min: usize = 0,

            /// True if this has been tripped at least once.
            tripped: bool = false,
        };

        /// Check for a failure at the given failure point. These should
        /// be placed directly before the `try` operation that may fail.
        pub fn check(point: FailPoint) callconv(callingConvention()) Error!void {
            if (comptime !enabled) return;
            return checkConstrained(point, Error);
        }

        /// Same as check but allows specifying a custom error type for the
        /// return value. This must be a subset of the module's Error type
        /// and will produce a runtime error if the configured tripwire
        /// error can't be cast to the ConstrainedError type.
        pub fn checkConstrained(
            point: FailPoint,
            comptime ConstrainedError: type,
        ) callconv(callingConvention()) ConstrainedError!void {
            if (comptime !enabled) return;
            const tripwire = tripwires.getPtr(point) orelse return;
            tripwire.reached += 1;
            if (tripwire.reached <= tripwire.min) return;
            tripwire.tripped = true;
            return tripwire.err;
        }

        /// Mark a failure point to always trip with the given error.
        pub fn errorAlways(point: FailPoint, err: Error) void {
            errorAfter(point, err, 0);
        }

        /// Mark a failure point to trip with the given error after
        /// the failure point is reached at least `min` times. A value of
        /// zero is equivalent to `errorAlways`.
        pub fn errorAfter(point: FailPoint, err: Error, min: usize) void {
            tripwires.put(point, .{ .err = err, .min = min });
        }

        /// Ends the tripwire session. This will raise an error if there
        /// were untripped error expectations. The reset mode specifies
        /// whether expectations are reset too. Expectations are always reset,
        /// even if this returns an error.
        pub fn end(reset_mode: enum { reset, retain }) error{UntrippedError}!void {
            var untripped: bool = false;
            var iter = tripwires.iterator();
            while (iter.next()) |entry| {
                if (!entry.value.tripped) {
                    log.warn("untripped point={s}", .{@tagName(entry.key)});
                    untripped = true;
                }
            }

            switch (reset_mode) {
                .reset => reset(),
                .retain => {},
            }

            if (untripped) return error.UntrippedError;
        }

        /// Unset all the tripwires. You should usually call `end` instead.
        pub fn reset() void {
            tripwires = .{};
        }

        /// Our calling convention is inline if our tripwire module is
        /// NOT enabled, so that all calls to `check` are optimized away.
        fn callingConvention() std.builtin.CallingConvention {
            return if (!enabled) .@"inline" else .auto;
        }
    };
}

test {
    const io = module(enum {
        read,
        write,
    }, anyerror);

    // Reset should work
    try io.end(.reset);

    // By default, its pass-through
    try io.check(.read);

    // Always trip
    io.errorAlways(.read, error.OutOfMemory);
    try testing.expectError(
        error.OutOfMemory,
        io.check(.read),
    );
    // Happens again
    try testing.expectError(
        error.OutOfMemory,
        io.check(.read),
    );
    try io.end(.reset);
}

test "module as error set" {
    const io = module(enum { read, write }, @TypeOf((struct {
        fn func() error{ Foo, Bar }!void {
            return error.Foo;
        }
    }).func));
    try io.end(.reset);
}

test "errorAfter" {
    const io = module(enum { read, write }, anyerror);
    // Trip after 2 calls (on the 3rd call)
    io.errorAfter(.read, error.OutOfMemory, 2);

    // First two calls succeed
    try io.check(.read);
    try io.check(.read);

    // Third call and on trips
    try testing.expectError(error.OutOfMemory, io.check(.read));
    try testing.expectError(error.OutOfMemory, io.check(.read));

    try io.end(.reset);
}

test "errorAfter untripped error if min not reached" {
    const io = module(enum { read }, anyerror);
    io.errorAfter(.read, error.OutOfMemory, 2);
    // Only call once, not enough to trip
    try io.check(.read);
    // end should fail because tripwire was set but never tripped
    try testing.expectError(
        error.UntrippedError,
        io.end(.reset),
    );
}