Invalid key sequence does not encode if a catch_all has ignore (#10038)

This adds some new special case handling for key sequences when an
unbound keyboard input is received. If the current keybinding set scope
(i.e. active tables) has a `catch_all` binding that would `ignore`
input, then the entire key sequence is dropped.

Normally, when an unbound key sequence is received, Ghostty encodes it
and sends it to the running program.

This special behavior is useful for things like Vim mode which have
`g>g` to scroll to top, and a `catch_all=ignore` to drop all other
input. If the user presses `g>h` (unbound), you don't want `gh` to show
up in your terminal input, because the `catch_all=ignore` indicates that
the user wants that mode to drop all unbound input.

src/Surface.zig

@@ -2826,14 +2826,21 @@ fn maybeHandleBinding(
 
             // No entry found. We need to encode everything up to this
             // point and send to the pty since we're in a sequence.
-            //
-            // We also ignore modifiers so that nested sequences such as
+
+            // We ignore modifiers so that nested sequences such as
             // ctrl+a>ctrl+b>c work.
-            if (!event.key.modifier()) {
-                // Encode everything up to this point
-                self.endKeySequence(.flush, .retain);
+            if (event.key.modifier()) return null;
+
+            // If we have a catch-all of ignore, then we special case our
+            // invalid sequence handling to ignore it.
+            if (self.catchAllIsIgnore()) {
+                self.endKeySequence(.drop, .retain);
+                return .ignored;
             }
 
+            // Encode everything up to this point
+            self.endKeySequence(.flush, .retain);
+
             return null;
         }
 
@@ -3037,6 +3044,34 @@ fn deactivateAllKeyTables(self: *Surface) !bool {
     return true;
 }
 
+/// This checks if the current keybinding sets have a catch_all binding
+/// with `ignore`. This is used to determine some special input cases.
+fn catchAllIsIgnore(self: *Surface) bool {
+    // Get our catch all
+    const entry: input.Binding.Set.Entry = entry: {
+        const trigger: input.Binding.Trigger = .{ .key = .catch_all };
+
+        const table_items = self.keyboard.table_stack.items;
+        for (0..table_items.len) |i| {
+            const rev_i: usize = table_items.len - 1 - i;
+            const entry = table_items[rev_i].set.get(trigger) orelse continue;
+            break :entry entry;
+        }
+
+        break :entry self.config.keybind.set.get(trigger) orelse
+            return false;
+    };
+
+    // We have a catch-all entry, see if its an ignore
+    return switch (entry.value_ptr.*) {
+        .leader => false,
+        .leaf => |leaf| leaf.action == .ignore,
+        .leaf_chained => |leaf| chained: for (leaf.actions.items) |action| {
+            if (action == .ignore) break :chained true;
+        } else false,
+    };
+}
+
 const KeySequenceQueued = enum { flush, drop };
 const KeySequenceMemory = enum { retain, free };
 
@@ -3065,23 +3100,26 @@ fn endKeySequence(
     // the set we look at to the root set.
     self.keyboard.sequence_set = null;
 
-    if (self.keyboard.sequence_queued.items.len > 0) {
-        switch (action) {
-            .flush => for (self.keyboard.sequence_queued.items) |write_req| {
-                self.queueIo(switch (write_req) {
-                    .small => |v| .{ .write_small = v },
-                    .stable => |v| .{ .write_stable = v },
-                    .alloc => |v| .{ .write_alloc = v },
-                }, .unlocked);
-            },
+    // If we have no queued data, there is nothing else to do.
+    if (self.keyboard.sequence_queued.items.len == 0) return;
 
-            .drop => for (self.keyboard.sequence_queued.items) |req| req.deinit(),
-        }
+    // Run the proper action first
+    switch (action) {
+        .flush => for (self.keyboard.sequence_queued.items) |write_req| {
+            self.queueIo(switch (write_req) {
+                .small => |v| .{ .write_small = v },
+                .stable => |v| .{ .write_stable = v },
+                .alloc => |v| .{ .write_alloc = v },
+            }, .unlocked);
+        },
 
-        switch (mem) {
-            .free => self.keyboard.sequence_queued.clearAndFree(self.alloc),
-            .retain => self.keyboard.sequence_queued.clearRetainingCapacity(),
-        }
+        .drop => for (self.keyboard.sequence_queued.items) |req| req.deinit(),
+    }
+
+    // Memory handling of the sequence after the action
+    switch (mem) {
+        .free => self.keyboard.sequence_queued.clearAndFree(self.alloc),
+        .retain => self.keyboard.sequence_queued.clearRetainingCapacity(),
     }
 }
 

src/config/Config.zig

@@ -1521,6 +1521,11 @@ class: ?[:0]const u8 = null,
 ///     specifically output that key (e.g. `ctrl+a>ctrl+a=text:foo`) or
 ///     press an unbound key which will send both keys to the program.
 ///
+///   * If an unbound key is pressed during a sequence and a `catch_all`
+///     binding exists that would `ignore` the input, the entire sequence
+///     is dropped and nothing happens. Otherwise, the entire sequence is
+///     encoded and sent to the running program as if no keybind existed.
+///
 ///   * If a prefix in a sequence is previously bound, the sequence will
 ///     override the previous binding. For example, if `ctrl+a` is bound to
 ///     `new_window` and `ctrl+a>n` is bound to `new_tab`, pressing `ctrl+a`